2024-11-25 03:13:39,754 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 03:13:39,768 main DEBUG Took 0.011624 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 03:13:39,768 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 03:13:39,769 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 03:13:39,769 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 03:13:39,771 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,778 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 03:13:39,792 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,794 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,795 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,797 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,798 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,799 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,800 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,801 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,802 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,803 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,804 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:13:39,806 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,806 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 03:13:39,808 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:13:39,810 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 03:13:39,813 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 03:13:39,813 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 03:13:39,815 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 03:13:39,815 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 03:13:39,823 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 03:13:39,826 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 03:13:39,827 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 03:13:39,828 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 03:13:39,828 main DEBUG createAppenders(={Console}) 2024-11-25 03:13:39,829 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-25 03:13:39,829 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 03:13:39,830 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-25 03:13:39,830 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 03:13:39,831 main DEBUG OutputStream closed 2024-11-25 03:13:39,831 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 03:13:39,831 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 03:13:39,832 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-25 03:13:39,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 03:13:39,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 03:13:39,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 03:13:39,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 03:13:39,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 03:13:39,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 03:13:39,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 03:13:39,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 03:13:39,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 03:13:39,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 03:13:39,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 03:13:39,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 03:13:39,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 03:13:39,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 03:13:39,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 03:13:39,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 03:13:39,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 03:13:39,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 03:13:39,918 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 03:13:39,919 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-25 03:13:39,919 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 03:13:39,920 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-25T03:13:40,164 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd 2024-11-25 03:13:40,167 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 03:13:40,167 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T03:13:40,176 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-25T03:13:40,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T03:13:40,222 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a, deleteOnExit=true 2024-11-25T03:13:40,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T03:13:40,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/test.cache.data in system properties and HBase conf 2024-11-25T03:13:40,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T03:13:40,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/hadoop.log.dir in system properties and HBase conf 2024-11-25T03:13:40,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T03:13:40,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T03:13:40,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T03:13:40,317 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T03:13:40,401 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:13:40,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:13:40,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:13:40,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T03:13:40,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:13:40,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T03:13:40,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T03:13:40,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:13:40,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:13:40,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T03:13:40,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/nfs.dump.dir in system properties and HBase conf 2024-11-25T03:13:40,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/java.io.tmpdir in system properties and HBase conf 2024-11-25T03:13:40,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:13:40,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T03:13:40,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T03:13:41,399 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T03:13:41,478 INFO [Time-limited test {}] log.Log(170): Logging initialized @2488ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T03:13:41,557 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:13:41,617 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:13:41,641 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:13:41,642 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:13:41,643 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:13:41,657 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:13:41,659 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:13:41,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:13:41,865 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/java.io.tmpdir/jetty-localhost-33351-hadoop-hdfs-3_4_1-tests_jar-_-any-637051332805784161/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:13:41,872 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33351} 2024-11-25T03:13:41,872 INFO [Time-limited test {}] server.Server(415): Started @2883ms 2024-11-25T03:13:42,275 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:13:42,282 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:13:42,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:13:42,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:13:42,283 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:13:42,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:13:42,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:13:42,405 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/java.io.tmpdir/jetty-localhost-38717-hadoop-hdfs-3_4_1-tests_jar-_-any-16451025122745582560/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:13:42,406 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:38717} 2024-11-25T03:13:42,406 INFO [Time-limited test {}] server.Server(415): Started @3417ms 2024-11-25T03:13:42,461 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:13:42,977 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/data/data1/current/BP-37819047-172.17.0.2-1732504421123/current, will proceed with Du for space computation calculation, 2024-11-25T03:13:42,977 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/data/data2/current/BP-37819047-172.17.0.2-1732504421123/current, will proceed with Du for space computation calculation, 2024-11-25T03:13:43,041 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:13:43,101 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x121976196385204b with lease ID 0xbcac6bbafcfa82f1: Processing first storage report for DS-30d57bf0-ea7c-4881-b572-353732cea40e from datanode DatanodeRegistration(127.0.0.1:42607, datanodeUuid=8fb8ee07-1675-42fa-a32b-ac65b0099908, infoPort=39637, infoSecurePort=0, ipcPort=35467, storageInfo=lv=-57;cid=testClusterID;nsid=38856446;c=1732504421123) 2024-11-25T03:13:43,104 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x121976196385204b with lease ID 0xbcac6bbafcfa82f1: from storage DS-30d57bf0-ea7c-4881-b572-353732cea40e node DatanodeRegistration(127.0.0.1:42607, datanodeUuid=8fb8ee07-1675-42fa-a32b-ac65b0099908, infoPort=39637, infoSecurePort=0, ipcPort=35467, storageInfo=lv=-57;cid=testClusterID;nsid=38856446;c=1732504421123), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-25T03:13:43,105 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x121976196385204b with lease ID 0xbcac6bbafcfa82f1: Processing first storage report for DS-5e2b9c00-4d3a-423c-8d47-4b8d6458fa90 from datanode DatanodeRegistration(127.0.0.1:42607, datanodeUuid=8fb8ee07-1675-42fa-a32b-ac65b0099908, infoPort=39637, infoSecurePort=0, ipcPort=35467, storageInfo=lv=-57;cid=testClusterID;nsid=38856446;c=1732504421123) 2024-11-25T03:13:43,105 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x121976196385204b with lease ID 0xbcac6bbafcfa82f1: from storage DS-5e2b9c00-4d3a-423c-8d47-4b8d6458fa90 node DatanodeRegistration(127.0.0.1:42607, datanodeUuid=8fb8ee07-1675-42fa-a32b-ac65b0099908, infoPort=39637, infoSecurePort=0, ipcPort=35467, storageInfo=lv=-57;cid=testClusterID;nsid=38856446;c=1732504421123), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:13:43,186 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd 2024-11-25T03:13:43,281 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/zookeeper_0, clientPort=58288, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:13:43,292 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58288 2024-11-25T03:13:43,305 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:13:43,309 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:13:43,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:13:44,142 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf with version=8 2024-11-25T03:13:44,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/hbase-staging 2024-11-25T03:13:44,258 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T03:13:44,517 INFO [Time-limited test {}] client.ConnectionUtils(128): master/38a5d2077268:0 server-side Connection retries=45 2024-11-25T03:13:44,528 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:13:44,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:13:44,533 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:13:44,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:13:44,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:13:44,686 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:13:44,765 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T03:13:44,776 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T03:13:44,781 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:13:44,814 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3080 (auto-detected) 2024-11-25T03:13:44,816 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T03:13:44,844 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44623 2024-11-25T03:13:44,875 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44623 connecting to ZooKeeper ensemble=127.0.0.1:58288 2024-11-25T03:13:44,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:446230x0, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:13:44,927 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44623-0x1014d1d75e90000 connected 2024-11-25T03:13:44,962 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:13:44,965 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:13:44,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:13:44,979 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf, hbase.cluster.distributed=false 2024-11-25T03:13:45,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:13:45,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44623 2024-11-25T03:13:45,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44623 2024-11-25T03:13:45,019 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44623 2024-11-25T03:13:45,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44623 2024-11-25T03:13:45,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44623 2024-11-25T03:13:45,154 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/38a5d2077268:0 server-side Connection retries=45 2024-11-25T03:13:45,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:13:45,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:13:45,158 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:13:45,158 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:13:45,158 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:13:45,162 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:13:45,165 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:13:45,169 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43357 2024-11-25T03:13:45,171 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43357 connecting to ZooKeeper ensemble=127.0.0.1:58288 2024-11-25T03:13:45,173 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:13:45,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:13:45,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433570x0, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:13:45,188 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43357-0x1014d1d75e90001 connected 2024-11-25T03:13:45,188 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:13:45,194 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:13:45,210 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:13:45,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:13:45,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:13:45,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43357 2024-11-25T03:13:45,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43357 2024-11-25T03:13:45,236 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43357 2024-11-25T03:13:45,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43357 2024-11-25T03:13:45,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43357 2024-11-25T03:13:45,258 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;38a5d2077268:44623 2024-11-25T03:13:45,261 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/38a5d2077268,44623,1732504424313 2024-11-25T03:13:45,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:13:45,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:13:45,271 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/38a5d2077268,44623,1732504424313 2024-11-25T03:13:45,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:45,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:13:45,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:45,295 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:13:45,296 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/38a5d2077268,44623,1732504424313 from backup master directory 2024-11-25T03:13:45,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:13:45,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/38a5d2077268,44623,1732504424313 2024-11-25T03:13:45,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:13:45,300 WARN [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:13:45,300 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=38a5d2077268,44623,1732504424313 2024-11-25T03:13:45,303 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T03:13:45,305 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T03:13:45,362 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/hbase.id] with ID: 8407abcc-f40b-4057-8321-91525263860d 2024-11-25T03:13:45,362 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/.tmp/hbase.id 2024-11-25T03:13:45,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:13:45,774 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/.tmp/hbase.id]:[hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/hbase.id] 2024-11-25T03:13:45,817 INFO [master/38a5d2077268:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:13:45,823 INFO [master/38a5d2077268:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:13:45,843 INFO [master/38a5d2077268:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-25T03:13:45,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:45,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:45,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:13:45,878 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:13:45,881 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:13:45,897 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:45,901 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:13:45,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:13:46,359 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store 2024-11-25T03:13:46,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:13:46,387 INFO [master/38a5d2077268:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T03:13:46,391 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:13:46,392 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:13:46,392 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:13:46,393 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:13:46,394 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:13:46,394 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:13:46,395 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:13:46,396 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732504426392Disabling compacts and flushes for region at 1732504426392Disabling writes for close at 1732504426394 (+2 ms)Writing region close event to WAL at 1732504426395 (+1 ms)Closed at 1732504426395 2024-11-25T03:13:46,399 WARN [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/.initializing 2024-11-25T03:13:46,399 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/WALs/38a5d2077268,44623,1732504424313 2024-11-25T03:13:46,410 INFO [master/38a5d2077268:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:13:46,430 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C44623%2C1732504424313, suffix=, logDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/WALs/38a5d2077268,44623,1732504424313, archiveDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/oldWALs, maxLogs=10 2024-11-25T03:13:46,459 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/WALs/38a5d2077268,44623,1732504424313/38a5d2077268%2C44623%2C1732504424313.1732504426435, exclude list is [], retry=0 2024-11-25T03:13:46,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42607,DS-30d57bf0-ea7c-4881-b572-353732cea40e,DISK] 2024-11-25T03:13:46,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T03:13:46,531 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/WALs/38a5d2077268,44623,1732504424313/38a5d2077268%2C44623%2C1732504424313.1732504426435 2024-11-25T03:13:46,533 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39637:39637)] 2024-11-25T03:13:46,534 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:13:46,534 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:13:46,540 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,541 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:13:46,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:46,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:46,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:13:46,649 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:46,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:13:46,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:13:46,654 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:46,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:13:46,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:13:46,659 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:46,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:13:46,661 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,665 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,666 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,673 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,674 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,678 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:13:46,683 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:13:46,689 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:13:46,690 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61882565, jitterRate=-0.07787792384624481}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:13:46,696 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732504426561Initializing all the Stores at 1732504426564 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732504426565 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504426566 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504426566Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504426566Cleaning up temporary data from old regions at 1732504426674 (+108 ms)Region opened successfully at 1732504426696 (+22 ms) 2024-11-25T03:13:46,698 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:13:46,745 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79110a47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=38a5d2077268/172.17.0.2:0 2024-11-25T03:13:46,787 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:13:46,799 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T03:13:46,799 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T03:13:46,802 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:13:46,804 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T03:13:46,809 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T03:13:46,809 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:13:46,836 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:13:46,845 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:13:46,848 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:13:46,850 INFO [master/38a5d2077268:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:13:46,852 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:13:46,854 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:13:46,857 INFO [master/38a5d2077268:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:13:46,860 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:13:46,862 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:13:46,863 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:13:46,865 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:13:46,883 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:13:46,884 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:13:46,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:13:46,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:13:46,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:46,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:46,892 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=38a5d2077268,44623,1732504424313, sessionid=0x1014d1d75e90000, setting cluster-up flag (Was=false) 2024-11-25T03:13:46,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:46,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:46,910 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:13:46,912 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=38a5d2077268,44623,1732504424313 2024-11-25T03:13:46,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:46,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:46,923 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:13:46,924 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=38a5d2077268,44623,1732504424313 2024-11-25T03:13:46,931 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:13:46,948 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(746): ClusterId : 8407abcc-f40b-4057-8321-91525263860d 2024-11-25T03:13:46,951 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:13:46,956 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:13:46,956 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:13:46,960 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:13:46,960 DEBUG [RS:0;38a5d2077268:43357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546bf5c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=38a5d2077268/172.17.0.2:0 2024-11-25T03:13:46,977 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;38a5d2077268:43357 2024-11-25T03:13:46,980 INFO [RS:0;38a5d2077268:43357 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:13:46,980 INFO [RS:0;38a5d2077268:43357 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:13:46,980 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:13:46,983 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(2659): reportForDuty to master=38a5d2077268,44623,1732504424313 with port=43357, startcode=1732504425108 2024-11-25T03:13:46,995 DEBUG [RS:0;38a5d2077268:43357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:13:47,014 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:13:47,028 INFO [master/38a5d2077268:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:13:47,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49717, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:13:47,036 INFO [master/38a5d2077268:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:13:47,042 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44623 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:47,042 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 38a5d2077268,44623,1732504424313 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:13:47,050 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:13:47,051 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:13:47,051 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:13:47,051 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:13:47,052 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/38a5d2077268:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:13:47,052 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,052 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/38a5d2077268:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:13:47,052 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,057 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732504457057 2024-11-25T03:13:47,058 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:13:47,058 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:13:47,058 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:13:47,059 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:13:47,063 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:13:47,063 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:13:47,063 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:13:47,063 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:13:47,064 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:47,064 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:13:47,064 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,067 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:13:47,068 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:13:47,068 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:13:47,073 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:13:47,073 WARN [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T03:13:47,073 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:13:47,074 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:13:47,075 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732504427075,5,FailOnTimeoutGroup] 2024-11-25T03:13:47,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:13:47,076 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732504427076,5,FailOnTimeoutGroup] 2024-11-25T03:13:47,076 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,076 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:13:47,077 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,078 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:13:47,078 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,078 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:13:47,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:13:47,175 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(2659): reportForDuty to master=38a5d2077268,44623,1732504424313 with port=43357, startcode=1732504425108 2024-11-25T03:13:47,178 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44623 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 38a5d2077268,43357,1732504425108 2024-11-25T03:13:47,180 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44623 {}] master.ServerManager(517): Registering regionserver=38a5d2077268,43357,1732504425108 2024-11-25T03:13:47,190 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:13:47,191 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32883 2024-11-25T03:13:47,191 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:13:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:13:47,195 DEBUG [RS:0;38a5d2077268:43357 {}] zookeeper.ZKUtil(111): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/38a5d2077268,43357,1732504425108 2024-11-25T03:13:47,196 WARN [RS:0;38a5d2077268:43357 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:13:47,196 INFO [RS:0;38a5d2077268:43357 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:13:47,196 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/WALs/38a5d2077268,43357,1732504425108 2024-11-25T03:13:47,198 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [38a5d2077268,43357,1732504425108] 2024-11-25T03:13:47,223 INFO [RS:0;38a5d2077268:43357 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:13:47,239 INFO [RS:0;38a5d2077268:43357 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:13:47,244 INFO [RS:0;38a5d2077268:43357 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:13:47,245 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,246 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:13:47,251 INFO [RS:0;38a5d2077268:43357 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:13:47,253 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,253 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,254 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,254 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,254 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,254 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,254 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/38a5d2077268:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:13:47,254 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,254 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,255 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,255 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,255 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,255 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:13:47,255 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/38a5d2077268:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:13:47,255 DEBUG [RS:0;38a5d2077268:43357 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:13:47,257 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,257 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,257 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,257 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,257 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,258 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,43357,1732504425108-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:13:47,279 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:13:47,281 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,43357,1732504425108-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,281 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,282 INFO [RS:0;38a5d2077268:43357 {}] regionserver.Replication(171): 38a5d2077268,43357,1732504425108 started 2024-11-25T03:13:47,301 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:47,301 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1482): Serving as 38a5d2077268,43357,1732504425108, RpcServer on 38a5d2077268/172.17.0.2:43357, sessionid=0x1014d1d75e90001 2024-11-25T03:13:47,302 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:13:47,302 DEBUG [RS:0;38a5d2077268:43357 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 38a5d2077268,43357,1732504425108 2024-11-25T03:13:47,302 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '38a5d2077268,43357,1732504425108' 2024-11-25T03:13:47,303 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:13:47,304 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:13:47,304 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:13:47,304 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:13:47,305 DEBUG [RS:0;38a5d2077268:43357 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 38a5d2077268,43357,1732504425108 2024-11-25T03:13:47,305 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '38a5d2077268,43357,1732504425108' 2024-11-25T03:13:47,305 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:13:47,305 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:13:47,306 DEBUG [RS:0;38a5d2077268:43357 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:13:47,306 INFO [RS:0;38a5d2077268:43357 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:13:47,306 INFO [RS:0;38a5d2077268:43357 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:13:47,412 INFO [RS:0;38a5d2077268:43357 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:13:47,415 INFO [RS:0;38a5d2077268:43357 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C43357%2C1732504425108, suffix=, logDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/WALs/38a5d2077268,43357,1732504425108, archiveDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/oldWALs, maxLogs=32 2024-11-25T03:13:47,433 DEBUG [RS:0;38a5d2077268:43357 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/WALs/38a5d2077268,43357,1732504425108/38a5d2077268%2C43357%2C1732504425108.1732504427418, exclude list is [], retry=0 2024-11-25T03:13:47,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42607,DS-30d57bf0-ea7c-4881-b572-353732cea40e,DISK] 2024-11-25T03:13:47,441 INFO [RS:0;38a5d2077268:43357 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/WALs/38a5d2077268,43357,1732504425108/38a5d2077268%2C43357%2C1732504425108.1732504427418 2024-11-25T03:13:47,442 DEBUG [RS:0;38a5d2077268:43357 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39637:39637)] 2024-11-25T03:13:47,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:13:47,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:13:47,498 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:13:47,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:47,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:47,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:13:47,501 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:13:47,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:47,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:47,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:13:47,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:13:47,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:47,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:47,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:13:47,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:13:47,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:47,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:47,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:13:47,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740 2024-11-25T03:13:47,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740 2024-11-25T03:13:47,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:13:47,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:13:47,516 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:13:47,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:13:47,523 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:13:47,524 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66225177, jitterRate=-0.013167962431907654}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:13:47,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732504427493Initializing all the Stores at 1732504427495 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732504427495Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732504427495Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504427495Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732504427495Cleaning up temporary data from old regions at 1732504427515 (+20 ms)Region opened successfully at 1732504427527 (+12 ms) 2024-11-25T03:13:47,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:13:47,528 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:13:47,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:13:47,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:13:47,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:13:47,530 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:13:47,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732504427528Disabling compacts and flushes for region at 1732504427528Disabling writes for close at 1732504427528Writing region close event to WAL at 1732504427529 (+1 ms)Closed at 1732504427530 (+1 ms) 2024-11-25T03:13:47,534 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:13:47,534 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:13:47,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:13:47,550 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:13:47,553 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:13:47,706 DEBUG [38a5d2077268:44623 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T03:13:47,720 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:13:47,728 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 38a5d2077268,43357,1732504425108, state=OPENING 2024-11-25T03:13:47,733 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:13:47,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:47,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:13:47,736 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:13:47,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:13:47,738 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:13:47,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:13:47,919 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:13:47,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53969, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:13:47,935 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:13:47,935 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:13:47,936 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:13:47,940 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C43357%2C1732504425108.meta, suffix=.meta, logDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/WALs/38a5d2077268,43357,1732504425108, archiveDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/oldWALs, maxLogs=32 2024-11-25T03:13:47,961 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/WALs/38a5d2077268,43357,1732504425108/38a5d2077268%2C43357%2C1732504425108.meta.1732504427943.meta, exclude list is [], retry=0 2024-11-25T03:13:47,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42607,DS-30d57bf0-ea7c-4881-b572-353732cea40e,DISK] 2024-11-25T03:13:47,969 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/WALs/38a5d2077268,43357,1732504425108/38a5d2077268%2C43357%2C1732504425108.meta.1732504427943.meta 2024-11-25T03:13:47,972 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39637:39637)] 2024-11-25T03:13:47,973 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:13:47,975 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:13:47,979 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:13:47,985 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:13:47,990 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:13:47,990 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:13:47,991 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:13:47,991 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:13:48,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:13:48,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:13:48,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:48,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:13:48,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:13:48,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:48,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:13:48,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:13:48,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:48,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:13:48,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:13:48,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:13:48,013 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:13:48,015 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740 2024-11-25T03:13:48,017 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740 2024-11-25T03:13:48,020 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:13:48,020 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:13:48,021 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:13:48,024 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:13:48,026 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71023227, jitterRate=0.05832855403423309}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:13:48,026 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:13:48,028 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732504427992Writing region info on filesystem at 1732504427992Initializing all the Stores at 1732504427994 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732504427994Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732504428000 (+6 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504428000Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732504428000Cleaning up temporary data from old regions at 1732504428020 (+20 ms)Running coprocessor post-open hooks at 1732504428026 (+6 ms)Region opened successfully at 1732504428028 (+2 ms) 2024-11-25T03:13:48,036 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732504427909 2024-11-25T03:13:48,051 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:13:48,051 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:13:48,053 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:13:48,056 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 38a5d2077268,43357,1732504425108, state=OPEN 2024-11-25T03:13:48,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:13:48,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:13:48,065 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:13:48,065 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:13:48,066 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:48,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:13:48,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=38a5d2077268,43357,1732504425108 in 326 msec 2024-11-25T03:13:48,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:13:48,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 535 msec 2024-11-25T03:13:48,083 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:13:48,084 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:13:48,113 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:48,114 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:48,137 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:48,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47079, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:48,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2050 sec 2024-11-25T03:13:48,170 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732504428169, completionTime=-1 2024-11-25T03:13:48,173 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T03:13:48,174 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:13:48,236 INFO [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T03:13:48,236 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732504488236 2024-11-25T03:13:48,236 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732504548236 2024-11-25T03:13:48,236 INFO [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 62 msec 2024-11-25T03:13:48,239 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,44623,1732504424313-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:48,239 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,44623,1732504424313-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:48,240 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,44623,1732504424313-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:48,241 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-38a5d2077268:44623, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:48,242 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:48,248 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:48,254 DEBUG [master/38a5d2077268:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:13:48,278 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.978sec 2024-11-25T03:13:48,279 INFO [master/38a5d2077268:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:13:48,281 INFO [master/38a5d2077268:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:13:48,281 INFO [master/38a5d2077268:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:13:48,282 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:13:48,282 INFO [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:13:48,283 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,44623,1732504424313-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:13:48,284 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,44623,1732504424313-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:13:48,292 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:13:48,293 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:13:48,294 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,44623,1732504424313-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:13:48,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:48,362 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T03:13:48,362 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T03:13:48,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:48,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:48,379 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6faf04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:48,398 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:48,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:48,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:48,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:48,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=38a5d2077268,44623,1732504424313 2024-11-25T03:13:48,458 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=7375 2024-11-25T03:13:48,468 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:13:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,44623,1732504424313 2024-11-25T03:13:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33f36750 2024-11-25T03:13:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:13:48,481 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:13:48,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:13:48,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:13:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:13:48,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:13:48,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-25T03:13:48,502 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:13:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:13:48,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741835_1011 (size=1061) 2024-11-25T03:13:48,532 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:13:48,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741836_1012 (size=53) 2024-11-25T03:13:48,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:13:48,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 159c9b03400b7fa4e53f5245dc0f2c9f, disabling compactions & flushes 2024-11-25T03:13:48,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. after waiting 0 ms 2024-11-25T03:13:48,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 159c9b03400b7fa4e53f5245dc0f2c9f: Waiting for close lock at 1732504428544Disabling compacts and flushes for region at 1732504428544Disabling writes for close at 1732504428544Writing region close event to WAL at 1732504428544Closed at 1732504428544 2024-11-25T03:13:48,547 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:13:48,553 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732504428547"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732504428547"}]},"ts":"1732504428547"} 2024-11-25T03:13:48,560 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:13:48,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:13:48,566 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504428562"}]},"ts":"1732504428562"} 2024-11-25T03:13:48,572 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:13:48,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, ASSIGN}] 2024-11-25T03:13:48,578 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, ASSIGN 2024-11-25T03:13:48,580 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, ASSIGN; state=OFFLINE, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=false 2024-11-25T03:13:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:13:48,732 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=159c9b03400b7fa4e53f5245dc0f2c9f, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:13:48,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, ASSIGN because future has completed 2024-11-25T03:13:48,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:13:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:13:48,903 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,904 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:13:48,904 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,904 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:13:48,905 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,905 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,907 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,910 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:13:48,911 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159c9b03400b7fa4e53f5245dc0f2c9f columnFamilyName A 2024-11-25T03:13:48,911 DEBUG [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,912 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.HStore(327): Store=159c9b03400b7fa4e53f5245dc0f2c9f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:13:48,912 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,914 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:13:48,914 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159c9b03400b7fa4e53f5245dc0f2c9f columnFamilyName B 2024-11-25T03:13:48,914 DEBUG [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,915 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.HStore(327): Store=159c9b03400b7fa4e53f5245dc0f2c9f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:13:48,915 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,917 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:13:48,918 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159c9b03400b7fa4e53f5245dc0f2c9f columnFamilyName C 2024-11-25T03:13:48,918 DEBUG [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:13:48,918 INFO [StoreOpener-159c9b03400b7fa4e53f5245dc0f2c9f-1 {}] regionserver.HStore(327): Store=159c9b03400b7fa4e53f5245dc0f2c9f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:13:48,919 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,919 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,921 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,922 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,924 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,924 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,925 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:13:48,928 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,932 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:13:48,932 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 159c9b03400b7fa4e53f5245dc0f2c9f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72772564, jitterRate=0.08439570665359497}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:13:48,933 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:48,934 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 159c9b03400b7fa4e53f5245dc0f2c9f: Running coprocessor pre-open hook at 1732504428905Writing region info on filesystem at 1732504428905Initializing all the Stores at 1732504428906 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504428906Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504428907 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504428907Cleaning up temporary data from old regions at 1732504428924 (+17 ms)Running coprocessor post-open hooks at 1732504428933 (+9 ms)Region opened successfully at 1732504428933 2024-11-25T03:13:48,936 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., pid=6, masterSystemTime=1732504428896 2024-11-25T03:13:48,940 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,940 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:48,941 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=159c9b03400b7fa4e53f5245dc0f2c9f, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:13:48,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:13:48,947 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44623 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=38a5d2077268,43357,1732504425108, table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:13:48,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T03:13:48,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 in 206 msec 2024-11-25T03:13:48,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T03:13:48,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, ASSIGN in 378 msec 2024-11-25T03:13:48,959 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:13:48,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504428959"}]},"ts":"1732504428959"} 2024-11-25T03:13:48,963 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:13:48,965 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:13:48,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 475 msec 2024-11-25T03:13:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:13:49,142 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:13:49,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dbb004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,166 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24557cee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,169 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,176 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:13:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,44623,1732504424313 2024-11-25T03:13:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f152604 2024-11-25T03:13:49,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:13:49,183 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:13:49,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbf42c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,196 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cb0f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,199 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51278417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,203 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246c200c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,209 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6b9e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,214 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31dd1621, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,220 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18731e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,226 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750f449b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,230 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4712ff10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,233 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500e7ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,238 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14983026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,242 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce65c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,245 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477e84a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70eed1b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,252 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7466909a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9903db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:13:49,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:13:49,260 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:13:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:13:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:13:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634e521d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:13:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:13:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:13:49,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:13:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c086271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:13:49,270 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,273 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,277 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,278 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,281 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,282 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,284 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,286 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,289 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,289 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:13:49,290 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:13:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:13:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:13:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:13:49,310 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:13:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-25T03:13:49,320 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:13:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:13:49,323 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:13:49,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:13:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:49,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:13:49,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:49,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:49,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:49,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:49,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:49,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:13:49,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:13:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:49,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:49,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504489543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504489542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504489560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/1657a4c67ddc4c51836161bc0d80804c is 50, key is test_row_0/A:col10/1732504429291/Put/seqid=0 2024-11-25T03:13:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504489571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504489572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741837_1013 (size=12001) 2024-11-25T03:13:49,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/1657a4c67ddc4c51836161bc0d80804c 2024-11-25T03:13:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:13:49,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:13:49,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504489692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504489693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504489693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:49,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504489712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504489713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/4c312a25f6c744668e7a9cf22de433d2 is 50, key is test_row_0/B:col10/1732504429291/Put/seqid=0 2024-11-25T03:13:49,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:13:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741838_1014 (size=12001) 2024-11-25T03:13:49,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/4c312a25f6c744668e7a9cf22de433d2 2024-11-25T03:13:49,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:49,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:49,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:49,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504489911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504489912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8877e1f6602244768ef8f512e4b822b3 is 50, key is test_row_0/C:col10/1732504429291/Put/seqid=0 2024-11-25T03:13:49,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504489921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504489932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504489932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:13:49,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741839_1015 (size=12001) 2024-11-25T03:13:49,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8877e1f6602244768ef8f512e4b822b3 2024-11-25T03:13:49,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/1657a4c67ddc4c51836161bc0d80804c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1657a4c67ddc4c51836161bc0d80804c 2024-11-25T03:13:50,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1657a4c67ddc4c51836161bc0d80804c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T03:13:50,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/4c312a25f6c744668e7a9cf22de433d2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/4c312a25f6c744668e7a9cf22de433d2 2024-11-25T03:13:50,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:13:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:50,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:50,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:50,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/4c312a25f6c744668e7a9cf22de433d2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T03:13:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8877e1f6602244768ef8f512e4b822b3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8877e1f6602244768ef8f512e4b822b3 2024-11-25T03:13:50,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8877e1f6602244768ef8f512e4b822b3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T03:13:50,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 159c9b03400b7fa4e53f5245dc0f2c9f in 675ms, sequenceid=13, compaction requested=false 2024-11-25T03:13:50,072 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T03:13:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:50,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:13:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:50,195 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T03:13:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/211b1461f4de41e9b6e569d1d745b508 is 50, key is test_row_0/A:col10/1732504429562/Put/seqid=0 2024-11-25T03:13:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:50,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741840_1016 (size=12001) 2024-11-25T03:13:50,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504490263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504490265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504490268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504490270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504490273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504490381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504490382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504490391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504490395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504490394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:13:50,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504490591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504490592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504490600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504490611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504490611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:50,649 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/211b1461f4de41e9b6e569d1d745b508 2024-11-25T03:13:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/fadf92afd3f84902adc570df143943e4 is 50, key is test_row_0/B:col10/1732504429562/Put/seqid=0 2024-11-25T03:13:50,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741841_1017 (size=12001) 2024-11-25T03:13:50,729 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/fadf92afd3f84902adc570df143943e4 2024-11-25T03:13:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/403a2b81f2b44f1f8c132d8aa43cc54c is 50, key is test_row_0/C:col10/1732504429562/Put/seqid=0 2024-11-25T03:13:50,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741842_1018 (size=12001) 2024-11-25T03:13:50,793 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/403a2b81f2b44f1f8c132d8aa43cc54c 2024-11-25T03:13:50,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/211b1461f4de41e9b6e569d1d745b508 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/211b1461f4de41e9b6e569d1d745b508 2024-11-25T03:13:50,824 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/211b1461f4de41e9b6e569d1d745b508, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T03:13:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/fadf92afd3f84902adc570df143943e4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/fadf92afd3f84902adc570df143943e4 2024-11-25T03:13:50,841 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/fadf92afd3f84902adc570df143943e4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T03:13:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/403a2b81f2b44f1f8c132d8aa43cc54c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/403a2b81f2b44f1f8c132d8aa43cc54c 2024-11-25T03:13:50,858 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/403a2b81f2b44f1f8c132d8aa43cc54c, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T03:13:50,860 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 159c9b03400b7fa4e53f5245dc0f2c9f in 666ms, sequenceid=33, compaction requested=false 2024-11-25T03:13:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-25T03:13:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-25T03:13:50,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T03:13:50,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-25T03:13:50,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5620 sec 2024-11-25T03:13:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:50,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:13:50,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:50,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/a9a52a3cde8442a0a0ce2066718c9e96 is 50, key is test_row_0/A:col10/1732504430268/Put/seqid=0 2024-11-25T03:13:50,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741843_1019 (size=12001) 2024-11-25T03:13:50,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/a9a52a3cde8442a0a0ce2066718c9e96 2024-11-25T03:13:50,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/0c9f9bf207a442e3be4896b94fe7c2bd is 50, key is test_row_0/B:col10/1732504430268/Put/seqid=0 2024-11-25T03:13:51,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741844_1020 (size=12001) 2024-11-25T03:13:51,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/0c9f9bf207a442e3be4896b94fe7c2bd 2024-11-25T03:13:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504490990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1b1bcee6a57b46aca04cec80004e142b is 50, key is test_row_0/C:col10/1732504430268/Put/seqid=0 2024-11-25T03:13:51,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741845_1021 (size=12001) 2024-11-25T03:13:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504491024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504490999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504491060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504491060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504491140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504491170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504491170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504491181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504491181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504491351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504491381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504491381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504491391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504491391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1b1bcee6a57b46aca04cec80004e142b 2024-11-25T03:13:51,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/a9a52a3cde8442a0a0ce2066718c9e96 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/a9a52a3cde8442a0a0ce2066718c9e96 2024-11-25T03:13:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:13:51,471 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:13:51,472 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:13:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-25T03:13:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:13:51,476 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:13:51,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:13:51,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:13:51,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/a9a52a3cde8442a0a0ce2066718c9e96, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:13:51,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/0c9f9bf207a442e3be4896b94fe7c2bd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/0c9f9bf207a442e3be4896b94fe7c2bd 2024-11-25T03:13:51,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/0c9f9bf207a442e3be4896b94fe7c2bd, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:13:51,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1b1bcee6a57b46aca04cec80004e142b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b1bcee6a57b46aca04cec80004e142b 2024-11-25T03:13:51,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b1bcee6a57b46aca04cec80004e142b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:13:51,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for 159c9b03400b7fa4e53f5245dc0f2c9f in 610ms, sequenceid=49, compaction requested=true 2024-11-25T03:13:51,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:51,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:13:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:51,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:13:51,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:51,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:13:51,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:51,541 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:51,541 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:51,547 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:51,549 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:13:51,551 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:51,552 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/4c312a25f6c744668e7a9cf22de433d2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/fadf92afd3f84902adc570df143943e4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/0c9f9bf207a442e3be4896b94fe7c2bd] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.2 K 2024-11-25T03:13:51,554 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c312a25f6c744668e7a9cf22de433d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732504429291 2024-11-25T03:13:51,557 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting fadf92afd3f84902adc570df143943e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732504429508 2024-11-25T03:13:51,558 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9f9bf207a442e3be4896b94fe7c2bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504430268 2024-11-25T03:13:51,560 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:51,560 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:13:51,560 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:51,561 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1657a4c67ddc4c51836161bc0d80804c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/211b1461f4de41e9b6e569d1d745b508, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/a9a52a3cde8442a0a0ce2066718c9e96] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.2 K 2024-11-25T03:13:51,562 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1657a4c67ddc4c51836161bc0d80804c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732504429291 2024-11-25T03:13:51,563 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 211b1461f4de41e9b6e569d1d745b508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732504429508 2024-11-25T03:13:51,564 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a52a3cde8442a0a0ce2066718c9e96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504430268 2024-11-25T03:13:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:13:51,600 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:51,601 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:51,602 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/f3ae286bdabf4954a1c1518481ec511a is 50, key is test_row_0/A:col10/1732504430268/Put/seqid=0 2024-11-25T03:13:51,602 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3bdfd9911bcc425ab268f25d6262c6ea is 50, key is test_row_0/B:col10/1732504430268/Put/seqid=0 2024-11-25T03:13:51,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741847_1023 (size=12104) 2024-11-25T03:13:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741846_1022 (size=12104) 2024-11-25T03:13:51,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T03:13:51,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:51,634 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:13:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:51,639 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3bdfd9911bcc425ab268f25d6262c6ea as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3bdfd9911bcc425ab268f25d6262c6ea 2024-11-25T03:13:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4611b49c14cc4873ab00342b7b61cc03 is 50, key is test_row_0/A:col10/1732504430952/Put/seqid=0 2024-11-25T03:13:51,671 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into 3bdfd9911bcc425ab268f25d6262c6ea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:51,671 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:51,684 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504431540; duration=0sec 2024-11-25T03:13:51,685 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:51,685 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:13:51,686 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:51,689 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:51,689 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:13:51,689 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:51,690 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8877e1f6602244768ef8f512e4b822b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/403a2b81f2b44f1f8c132d8aa43cc54c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b1bcee6a57b46aca04cec80004e142b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.2 K 2024-11-25T03:13:51,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741848_1024 (size=12001) 2024-11-25T03:13:51,694 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8877e1f6602244768ef8f512e4b822b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732504429291 2024-11-25T03:13:51,695 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 403a2b81f2b44f1f8c132d8aa43cc54c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732504429508 2024-11-25T03:13:51,695 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4611b49c14cc4873ab00342b7b61cc03 2024-11-25T03:13:51,696 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1bcee6a57b46aca04cec80004e142b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504430268 2024-11-25T03:13:51,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/dc3ca30e58b04b14ae62c6e7988708af is 50, key is test_row_0/B:col10/1732504430952/Put/seqid=0 2024-11-25T03:13:51,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504491729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504491729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,739 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:51,740 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/51891bf05ea943a481faa85b276ef7cc is 50, key is test_row_0/C:col10/1732504430268/Put/seqid=0 2024-11-25T03:13:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504491732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504491736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504491737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741849_1025 (size=12001) 2024-11-25T03:13:51,784 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/dc3ca30e58b04b14ae62c6e7988708af 2024-11-25T03:13:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:13:51,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/999096c7e6e0414189067ed877340314 is 50, key is test_row_0/C:col10/1732504430952/Put/seqid=0 2024-11-25T03:13:51,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741850_1026 (size=12104) 2024-11-25T03:13:51,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504491840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,848 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/51891bf05ea943a481faa85b276ef7cc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/51891bf05ea943a481faa85b276ef7cc 2024-11-25T03:13:51,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504491850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504491861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504491861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504491861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:51,872 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into 51891bf05ea943a481faa85b276ef7cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:51,872 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,873 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504431541; duration=0sec 2024-11-25T03:13:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:51,873 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:51,873 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:13:51,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741851_1027 (size=12001) 2024-11-25T03:13:51,877 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/999096c7e6e0414189067ed877340314 2024-11-25T03:13:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4611b49c14cc4873ab00342b7b61cc03 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4611b49c14cc4873ab00342b7b61cc03 2024-11-25T03:13:51,903 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4611b49c14cc4873ab00342b7b61cc03, entries=150, sequenceid=67, filesize=11.7 K 2024-11-25T03:13:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/dc3ca30e58b04b14ae62c6e7988708af as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/dc3ca30e58b04b14ae62c6e7988708af 2024-11-25T03:13:51,919 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/dc3ca30e58b04b14ae62c6e7988708af, entries=150, sequenceid=67, filesize=11.7 K 2024-11-25T03:13:51,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/999096c7e6e0414189067ed877340314 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/999096c7e6e0414189067ed877340314 2024-11-25T03:13:51,942 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/999096c7e6e0414189067ed877340314, entries=150, sequenceid=67, filesize=11.7 K 2024-11-25T03:13:51,945 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 159c9b03400b7fa4e53f5245dc0f2c9f in 312ms, sequenceid=67, compaction requested=false 2024-11-25T03:13:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-25T03:13:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-25T03:13:51,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-25T03:13:51,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 472 msec 2024-11-25T03:13:51,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 489 msec 2024-11-25T03:13:52,042 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/f3ae286bdabf4954a1c1518481ec511a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f3ae286bdabf4954a1c1518481ec511a 2024-11-25T03:13:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:52,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T03:13:52,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:52,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:52,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:52,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:52,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:52,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:52,058 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into f3ae286bdabf4954a1c1518481ec511a(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:52,058 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:52,058 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504431533; duration=0sec 2024-11-25T03:13:52,059 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:52,059 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:13:52,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/47f83ba0578b4f1c83093dd90fb1ca68 is 50, key is test_row_0/A:col10/1732504431727/Put/seqid=0 2024-11-25T03:13:52,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741852_1028 (size=14341) 2024-11-25T03:13:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:13:52,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/47f83ba0578b4f1c83093dd90fb1ca68 2024-11-25T03:13:52,102 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:13:52,103 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:13:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-25T03:13:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:13:52,107 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:13:52,108 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:13:52,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:13:52,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/70e824c2c6084d1d85b290b8c54a70eb is 50, key is test_row_0/B:col10/1732504431727/Put/seqid=0 2024-11-25T03:13:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741853_1029 (size=12001) 2024-11-25T03:13:52,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504492145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504492145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504492148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504492148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504492153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:13:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504492262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504492262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504492262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T03:13:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504492270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504492270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:13:52,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T03:13:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504492471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504492471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504492471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504492491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504492491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/70e824c2c6084d1d85b290b8c54a70eb 2024-11-25T03:13:52,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/079ff8cc819c41c98c653fe891c33a5d is 50, key is test_row_0/C:col10/1732504431727/Put/seqid=0 2024-11-25T03:13:52,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741854_1030 (size=12001) 2024-11-25T03:13:52,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/079ff8cc819c41c98c653fe891c33a5d 2024-11-25T03:13:52,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T03:13:52,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:52,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/47f83ba0578b4f1c83093dd90fb1ca68 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/47f83ba0578b4f1c83093dd90fb1ca68 2024-11-25T03:13:52,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/47f83ba0578b4f1c83093dd90fb1ca68, entries=200, sequenceid=86, filesize=14.0 K 2024-11-25T03:13:52,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/70e824c2c6084d1d85b290b8c54a70eb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/70e824c2c6084d1d85b290b8c54a70eb 2024-11-25T03:13:52,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/70e824c2c6084d1d85b290b8c54a70eb, entries=150, sequenceid=86, filesize=11.7 K 2024-11-25T03:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/079ff8cc819c41c98c653fe891c33a5d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/079ff8cc819c41c98c653fe891c33a5d 2024-11-25T03:13:52,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/079ff8cc819c41c98c653fe891c33a5d, entries=150, sequenceid=86, filesize=11.7 K 2024-11-25T03:13:52,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 159c9b03400b7fa4e53f5245dc0f2c9f in 569ms, sequenceid=86, compaction requested=true 2024-11-25T03:13:52,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:52,627 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:13:52,630 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:52,630 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:13:52,630 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,630 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f3ae286bdabf4954a1c1518481ec511a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4611b49c14cc4873ab00342b7b61cc03, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/47f83ba0578b4f1c83093dd90fb1ca68] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=37.5 K 2024-11-25T03:13:52,633 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ae286bdabf4954a1c1518481ec511a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504430268 2024-11-25T03:13:52,633 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4611b49c14cc4873ab00342b7b61cc03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732504430952 2024-11-25T03:13:52,634 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f83ba0578b4f1c83093dd90fb1ca68, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732504431711 2024-11-25T03:13:52,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:52,636 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:52,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:13:52,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:52,639 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:52,640 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:13:52,640 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,640 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3bdfd9911bcc425ab268f25d6262c6ea, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/dc3ca30e58b04b14ae62c6e7988708af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/70e824c2c6084d1d85b290b8c54a70eb] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.3 K 2024-11-25T03:13:52,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:13:52,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:52,641 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bdfd9911bcc425ab268f25d6262c6ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504430268 2024-11-25T03:13:52,642 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting dc3ca30e58b04b14ae62c6e7988708af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732504430952 2024-11-25T03:13:52,643 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e824c2c6084d1d85b290b8c54a70eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732504431711 2024-11-25T03:13:52,666 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:52,667 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ffec0c209bfa4e16a6dae6e569b8ffc6 is 50, key is test_row_0/B:col10/1732504431727/Put/seqid=0 2024-11-25T03:13:52,667 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:52,668 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/dd2123c81f4b4d77ae908317874c237a is 50, key is test_row_0/A:col10/1732504431727/Put/seqid=0 2024-11-25T03:13:52,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741855_1031 (size=12207) 2024-11-25T03:13:52,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741856_1032 (size=12207) 2024-11-25T03:13:52,724 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ffec0c209bfa4e16a6dae6e569b8ffc6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffec0c209bfa4e16a6dae6e569b8ffc6 2024-11-25T03:13:52,729 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/dd2123c81f4b4d77ae908317874c237a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/dd2123c81f4b4d77ae908317874c237a 2024-11-25T03:13:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:13:52,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T03:13:52,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,735 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T03:13:52,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:52,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:52,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:52,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:52,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:52,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:52,744 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into dd2123c81f4b4d77ae908317874c237a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:52,744 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:52,744 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504432627; duration=0sec 2024-11-25T03:13:52,744 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:52,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:13:52,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:52,745 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into ffec0c209bfa4e16a6dae6e569b8ffc6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:52,745 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:52,745 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504432636; duration=0sec 2024-11-25T03:13:52,745 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:52,746 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:13:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/fd160a6838aa4e64a6881b80a72a47ee is 50, key is test_row_0/A:col10/1732504432124/Put/seqid=0 2024-11-25T03:13:52,751 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:52,751 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:13:52,751 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:52,752 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/51891bf05ea943a481faa85b276ef7cc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/999096c7e6e0414189067ed877340314, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/079ff8cc819c41c98c653fe891c33a5d] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.3 K 2024-11-25T03:13:52,753 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51891bf05ea943a481faa85b276ef7cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504430268 2024-11-25T03:13:52,753 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 999096c7e6e0414189067ed877340314, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732504430952 2024-11-25T03:13:52,758 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 079ff8cc819c41c98c653fe891c33a5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732504431711 2024-11-25T03:13:52,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741857_1033 (size=12001) 2024-11-25T03:13:52,780 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/fd160a6838aa4e64a6881b80a72a47ee 2024-11-25T03:13:52,783 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:52,784 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1bd1db95ea05430ea73219331e3e9bb8 is 50, key is test_row_0/C:col10/1732504431727/Put/seqid=0 2024-11-25T03:13:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:52,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741858_1034 (size=12207) 2024-11-25T03:13:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/a9f1e59232e045af98af2941fa71770a is 50, key is test_row_0/B:col10/1732504432124/Put/seqid=0 2024-11-25T03:13:52,828 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1bd1db95ea05430ea73219331e3e9bb8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1bd1db95ea05430ea73219331e3e9bb8 2024-11-25T03:13:52,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504492818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504492822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504492826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504492831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504492829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,849 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into 1bd1db95ea05430ea73219331e3e9bb8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:52,849 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:52,849 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504432639; duration=0sec 2024-11-25T03:13:52,849 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:52,849 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:13:52,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741859_1035 (size=12001) 2024-11-25T03:13:52,888 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/a9f1e59232e045af98af2941fa71770a 2024-11-25T03:13:52,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/2ea5b64c5c9f4137b0f9a9835f55ecdb is 50, key is test_row_0/C:col10/1732504432124/Put/seqid=0 2024-11-25T03:13:52,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741860_1036 (size=12001) 2024-11-25T03:13:52,946 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/2ea5b64c5c9f4137b0f9a9835f55ecdb 2024-11-25T03:13:52,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504492941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/fd160a6838aa4e64a6881b80a72a47ee as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/fd160a6838aa4e64a6881b80a72a47ee 2024-11-25T03:13:52,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504492955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504492955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504492955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504492956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:52,969 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/fd160a6838aa4e64a6881b80a72a47ee, entries=150, sequenceid=102, filesize=11.7 K 2024-11-25T03:13:52,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/a9f1e59232e045af98af2941fa71770a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a9f1e59232e045af98af2941fa71770a 2024-11-25T03:13:52,986 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a9f1e59232e045af98af2941fa71770a, entries=150, sequenceid=102, filesize=11.7 K 2024-11-25T03:13:52,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/2ea5b64c5c9f4137b0f9a9835f55ecdb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/2ea5b64c5c9f4137b0f9a9835f55ecdb 2024-11-25T03:13:53,019 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/2ea5b64c5c9f4137b0f9a9835f55ecdb, entries=150, sequenceid=102, filesize=11.7 K 2024-11-25T03:13:53,022 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 159c9b03400b7fa4e53f5245dc0f2c9f in 286ms, sequenceid=102, compaction requested=false 2024-11-25T03:13:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-25T03:13:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-25T03:13:53,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T03:13:53,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 918 msec 2024-11-25T03:13:53,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 930 msec 2024-11-25T03:13:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:53,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:13:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:53,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/8462d576bb324b4cb34ad324b77d829a is 50, key is test_row_0/A:col10/1732504432822/Put/seqid=0 2024-11-25T03:13:53,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504493193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504493197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504493200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504493197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504493205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741861_1037 (size=14341) 2024-11-25T03:13:53,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/8462d576bb324b4cb34ad324b77d829a 2024-11-25T03:13:53,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/df58ee78a3434df7890fbe4eaa731aba is 50, key is test_row_0/B:col10/1732504432822/Put/seqid=0 2024-11-25T03:13:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:13:53,241 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:13:53,241 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:13:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-25T03:13:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:13:53,245 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:13:53,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:13:53,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:13:53,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741862_1038 (size=12001) 2024-11-25T03:13:53,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/df58ee78a3434df7890fbe4eaa731aba 2024-11-25T03:13:53,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/a2114f2058064228abbd0cf2d9bfea49 is 50, key is test_row_0/C:col10/1732504432822/Put/seqid=0 2024-11-25T03:13:53,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741863_1039 (size=12001) 2024-11-25T03:13:53,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/a2114f2058064228abbd0cf2d9bfea49 2024-11-25T03:13:53,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/8462d576bb324b4cb34ad324b77d829a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/8462d576bb324b4cb34ad324b77d829a 2024-11-25T03:13:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504493311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504493311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504493312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504493312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504493312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/8462d576bb324b4cb34ad324b77d829a, entries=200, sequenceid=122, filesize=14.0 K 2024-11-25T03:13:53,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/df58ee78a3434df7890fbe4eaa731aba as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/df58ee78a3434df7890fbe4eaa731aba 2024-11-25T03:13:53,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/df58ee78a3434df7890fbe4eaa731aba, entries=150, sequenceid=122, filesize=11.7 K 2024-11-25T03:13:53,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/a2114f2058064228abbd0cf2d9bfea49 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/a2114f2058064228abbd0cf2d9bfea49 2024-11-25T03:13:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:13:53,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/a2114f2058064228abbd0cf2d9bfea49, entries=150, sequenceid=122, filesize=11.7 K 2024-11-25T03:13:53,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 159c9b03400b7fa4e53f5245dc0f2c9f in 196ms, sequenceid=122, compaction requested=true 2024-11-25T03:13:53,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:53,360 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:53,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:13:53,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:53,361 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:53,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:13:53,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:53,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:13:53,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:53,362 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:53,363 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:13:53,363 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:53,363 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/dd2123c81f4b4d77ae908317874c237a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/fd160a6838aa4e64a6881b80a72a47ee, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/8462d576bb324b4cb34ad324b77d829a] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=37.6 K 2024-11-25T03:13:53,364 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd2123c81f4b4d77ae908317874c237a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732504431711 2024-11-25T03:13:53,364 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:53,364 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:13:53,364 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:53,365 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffec0c209bfa4e16a6dae6e569b8ffc6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a9f1e59232e045af98af2941fa71770a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/df58ee78a3434df7890fbe4eaa731aba] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.4 K 2024-11-25T03:13:53,365 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd160a6838aa4e64a6881b80a72a47ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732504432088 2024-11-25T03:13:53,367 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ffec0c209bfa4e16a6dae6e569b8ffc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732504431711 2024-11-25T03:13:53,367 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8462d576bb324b4cb34ad324b77d829a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732504432813 2024-11-25T03:13:53,368 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f1e59232e045af98af2941fa71770a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732504432088 2024-11-25T03:13:53,370 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting df58ee78a3434df7890fbe4eaa731aba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732504432813 2024-11-25T03:13:53,386 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:53,387 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/d850b17fbbd547aaad73e06e87804169 is 50, key is test_row_0/A:col10/1732504432822/Put/seqid=0 2024-11-25T03:13:53,395 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:53,396 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/497525ee439b47fd8398e8d3d6a325e8 is 50, key is test_row_0/B:col10/1732504432822/Put/seqid=0 2024-11-25T03:13:53,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T03:13:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:53,403 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:13:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:53,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741864_1040 (size=12309) 2024-11-25T03:13:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ef1b0dd7976c4e8eb13d8946ec699ae4 is 50, key is test_row_0/A:col10/1732504433192/Put/seqid=0 2024-11-25T03:13:53,434 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/d850b17fbbd547aaad73e06e87804169 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d850b17fbbd547aaad73e06e87804169 2024-11-25T03:13:53,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741865_1041 (size=12309) 2024-11-25T03:13:53,447 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/497525ee439b47fd8398e8d3d6a325e8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/497525ee439b47fd8398e8d3d6a325e8 2024-11-25T03:13:53,452 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into d850b17fbbd547aaad73e06e87804169(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:53,452 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:53,452 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504433360; duration=0sec 2024-11-25T03:13:53,452 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:53,452 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:13:53,452 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:53,457 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:53,458 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:13:53,459 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:53,460 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1bd1db95ea05430ea73219331e3e9bb8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/2ea5b64c5c9f4137b0f9a9835f55ecdb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/a2114f2058064228abbd0cf2d9bfea49] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.4 K 2024-11-25T03:13:53,462 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into 497525ee439b47fd8398e8d3d6a325e8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:53,462 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:53,462 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504433361; duration=0sec 2024-11-25T03:13:53,463 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:53,463 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:13:53,463 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bd1db95ea05430ea73219331e3e9bb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732504431711 2024-11-25T03:13:53,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741866_1042 (size=12151) 2024-11-25T03:13:53,469 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea5b64c5c9f4137b0f9a9835f55ecdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732504432088 2024-11-25T03:13:53,470 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2114f2058064228abbd0cf2d9bfea49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732504432813 2024-11-25T03:13:53,514 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:13:53,519 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:53,519 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c8f602b6706f4e7c8075f3412d88d2c5 is 50, key is test_row_0/C:col10/1732504432822/Put/seqid=0 2024-11-25T03:13:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:53,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:13:53,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741867_1043 (size=12309) 2024-11-25T03:13:53,595 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c8f602b6706f4e7c8075f3412d88d2c5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8f602b6706f4e7c8075f3412d88d2c5 2024-11-25T03:13:53,607 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into c8f602b6706f4e7c8075f3412d88d2c5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:53,608 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:53,608 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504433362; duration=0sec 2024-11-25T03:13:53,608 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:53,608 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:13:53,609 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:13:53,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504493607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504493610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504493610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504493612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504493615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504493731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504493732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504493731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504493732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504493733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,867 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ef1b0dd7976c4e8eb13d8946ec699ae4 2024-11-25T03:13:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:13:53,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/407afd90567e49528437378d959aeeaa is 50, key is test_row_0/B:col10/1732504433192/Put/seqid=0 2024-11-25T03:13:53,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741868_1044 (size=12151) 2024-11-25T03:13:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504493940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504493941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504493941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504493941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504493942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504494250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504494251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504494251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504494251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504494252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,328 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/407afd90567e49528437378d959aeeaa 2024-11-25T03:13:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/31d36055103b492bb7ef4289f37b656a is 50, key is test_row_0/C:col10/1732504433192/Put/seqid=0 2024-11-25T03:13:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:13:54,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741869_1045 (size=12151) 2024-11-25T03:13:54,396 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/31d36055103b492bb7ef4289f37b656a 2024-11-25T03:13:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ef1b0dd7976c4e8eb13d8946ec699ae4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef1b0dd7976c4e8eb13d8946ec699ae4 2024-11-25T03:13:54,416 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef1b0dd7976c4e8eb13d8946ec699ae4, entries=150, sequenceid=137, filesize=11.9 K 2024-11-25T03:13:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/407afd90567e49528437378d959aeeaa as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/407afd90567e49528437378d959aeeaa 2024-11-25T03:13:54,429 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/407afd90567e49528437378d959aeeaa, entries=150, sequenceid=137, filesize=11.9 K 2024-11-25T03:13:54,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/31d36055103b492bb7ef4289f37b656a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/31d36055103b492bb7ef4289f37b656a 2024-11-25T03:13:54,444 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/31d36055103b492bb7ef4289f37b656a, entries=150, sequenceid=137, filesize=11.9 K 2024-11-25T03:13:54,453 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 159c9b03400b7fa4e53f5245dc0f2c9f in 1046ms, sequenceid=137, compaction requested=false 2024-11-25T03:13:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:54,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-25T03:13:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-25T03:13:54,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T03:13:54,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2110 sec 2024-11-25T03:13:54,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2240 sec 2024-11-25T03:13:54,757 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:13:54,757 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:13:54,760 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:13:54,761 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:13:54,761 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T03:13:54,761 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T03:13:54,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T03:13:54,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T03:13:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:54,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:13:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:54,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:54,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:54,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:54,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:54,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/42f2f81997d945bfabf96ff4cad94c7c is 50, key is test_row_0/A:col10/1732504433611/Put/seqid=0 2024-11-25T03:13:54,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741870_1046 (size=14541) 2024-11-25T03:13:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504494792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504494792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504494795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504494795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504494800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504494910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504494911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504494911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504494911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504494911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504495121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504495121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504495131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504495131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504495131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/42f2f81997d945bfabf96ff4cad94c7c 2024-11-25T03:13:55,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/7a253de9d790446585bf960b47e2e961 is 50, key is test_row_0/B:col10/1732504433611/Put/seqid=0 2024-11-25T03:13:55,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741871_1047 (size=12151) 2024-11-25T03:13:55,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/7a253de9d790446585bf960b47e2e961 2024-11-25T03:13:55,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/4288ac7f75274a6fbfd11935fe48cb37 is 50, key is test_row_0/C:col10/1732504433611/Put/seqid=0 2024-11-25T03:13:55,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741872_1048 (size=12151) 2024-11-25T03:13:55,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/4288ac7f75274a6fbfd11935fe48cb37 2024-11-25T03:13:55,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/42f2f81997d945bfabf96ff4cad94c7c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/42f2f81997d945bfabf96ff4cad94c7c 2024-11-25T03:13:55,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/42f2f81997d945bfabf96ff4cad94c7c, entries=200, sequenceid=158, filesize=14.2 K 2024-11-25T03:13:55,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/7a253de9d790446585bf960b47e2e961 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7a253de9d790446585bf960b47e2e961 2024-11-25T03:13:55,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7a253de9d790446585bf960b47e2e961, entries=150, sequenceid=158, filesize=11.9 K 2024-11-25T03:13:55,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/4288ac7f75274a6fbfd11935fe48cb37 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/4288ac7f75274a6fbfd11935fe48cb37 2024-11-25T03:13:55,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/4288ac7f75274a6fbfd11935fe48cb37, entries=150, sequenceid=158, filesize=11.9 K 2024-11-25T03:13:55,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 159c9b03400b7fa4e53f5245dc0f2c9f in 525ms, sequenceid=158, compaction requested=true 2024-11-25T03:13:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:13:55,293 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:13:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:55,293 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:13:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:55,294 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:55,294 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:55,294 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:13:55,295 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,295 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:13:55,295 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d850b17fbbd547aaad73e06e87804169, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef1b0dd7976c4e8eb13d8946ec699ae4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/42f2f81997d945bfabf96ff4cad94c7c] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=38.1 K 2024-11-25T03:13:55,295 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,295 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/497525ee439b47fd8398e8d3d6a325e8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/407afd90567e49528437378d959aeeaa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7a253de9d790446585bf960b47e2e961] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.8 K 2024-11-25T03:13:55,295 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d850b17fbbd547aaad73e06e87804169, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732504432813 2024-11-25T03:13:55,296 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 497525ee439b47fd8398e8d3d6a325e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732504432813 2024-11-25T03:13:55,296 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1b0dd7976c4e8eb13d8946ec699ae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732504433192 2024-11-25T03:13:55,296 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 407afd90567e49528437378d959aeeaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732504433192 2024-11-25T03:13:55,297 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f2f81997d945bfabf96ff4cad94c7c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732504433595 2024-11-25T03:13:55,298 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a253de9d790446585bf960b47e2e961, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732504433595 2024-11-25T03:13:55,314 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:55,315 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/e9de676ee33a474a932919299e7233c1 is 50, key is test_row_0/A:col10/1732504433611/Put/seqid=0 2024-11-25T03:13:55,318 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:55,318 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/457a2ab9b81d41ae9f9adc080d69f433 is 50, key is test_row_0/B:col10/1732504433611/Put/seqid=0 2024-11-25T03:13:55,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741873_1049 (size=12561) 2024-11-25T03:13:55,344 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/e9de676ee33a474a932919299e7233c1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e9de676ee33a474a932919299e7233c1 2024-11-25T03:13:55,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741874_1050 (size=12561) 2024-11-25T03:13:55,358 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/457a2ab9b81d41ae9f9adc080d69f433 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/457a2ab9b81d41ae9f9adc080d69f433 2024-11-25T03:13:55,359 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into e9de676ee33a474a932919299e7233c1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:55,359 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:55,359 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504435292; duration=0sec 2024-11-25T03:13:55,360 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:55,360 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:13:55,360 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:55,363 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:55,364 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:13:55,364 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,364 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8f602b6706f4e7c8075f3412d88d2c5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/31d36055103b492bb7ef4289f37b656a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/4288ac7f75274a6fbfd11935fe48cb37] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=35.8 K 2024-11-25T03:13:55,366 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8f602b6706f4e7c8075f3412d88d2c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732504432813 2024-11-25T03:13:55,367 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31d36055103b492bb7ef4289f37b656a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732504433192 2024-11-25T03:13:55,369 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4288ac7f75274a6fbfd11935fe48cb37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732504433595 2024-11-25T03:13:55,371 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into 457a2ab9b81d41ae9f9adc080d69f433(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:55,371 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:55,371 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504435293; duration=0sec 2024-11-25T03:13:55,371 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:55,371 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:13:55,381 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:55,382 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/5369c9fb9e164a1d86324fc83e90e001 is 50, key is test_row_0/C:col10/1732504433611/Put/seqid=0 2024-11-25T03:13:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:13:55,393 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:13:55,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:13:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-25T03:13:55,398 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:13:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:13:55,400 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:13:55,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:13:55,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741875_1051 (size=12561) 2024-11-25T03:13:55,423 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/5369c9fb9e164a1d86324fc83e90e001 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/5369c9fb9e164a1d86324fc83e90e001 2024-11-25T03:13:55,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:13:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:55,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:55,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:55,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:55,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:55,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:55,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:55,436 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into 5369c9fb9e164a1d86324fc83e90e001(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:55,437 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:55,437 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504435293; duration=0sec 2024-11-25T03:13:55,437 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:55,437 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:13:55,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c6e91bd6512b45dc88267fefcce96a3d is 50, key is test_row_0/A:col10/1732504434796/Put/seqid=0 2024-11-25T03:13:55,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741876_1052 (size=16931) 2024-11-25T03:13:55,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c6e91bd6512b45dc88267fefcce96a3d 2024-11-25T03:13:55,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504495473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504495470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504495475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504495478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504495482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/107af241e0064457959f9e0198ead7a7 is 50, key is test_row_0/B:col10/1732504434796/Put/seqid=0 2024-11-25T03:13:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:13:55,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741877_1053 (size=12151) 2024-11-25T03:13:55,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/107af241e0064457959f9e0198ead7a7 2024-11-25T03:13:55,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/3126f9b5ef8e45f581eb0f9ad42d00e5 is 50, key is test_row_0/C:col10/1732504434796/Put/seqid=0 2024-11-25T03:13:55,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741878_1054 (size=12151) 2024-11-25T03:13:55,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T03:13:55,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:55,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504495590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504495592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504495592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504495592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504495592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T03:13:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:13:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504495800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504495801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504495801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504495801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504495801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:55,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T03:13:55,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:55,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:55,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:55,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/3126f9b5ef8e45f581eb0f9ad42d00e5 2024-11-25T03:13:55,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c6e91bd6512b45dc88267fefcce96a3d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6e91bd6512b45dc88267fefcce96a3d 2024-11-25T03:13:55,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6e91bd6512b45dc88267fefcce96a3d, entries=250, sequenceid=176, filesize=16.5 K 2024-11-25T03:13:55,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/107af241e0064457959f9e0198ead7a7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/107af241e0064457959f9e0198ead7a7 2024-11-25T03:13:55,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/107af241e0064457959f9e0198ead7a7, entries=150, sequenceid=176, filesize=11.9 K 2024-11-25T03:13:55,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/3126f9b5ef8e45f581eb0f9ad42d00e5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/3126f9b5ef8e45f581eb0f9ad42d00e5 2024-11-25T03:13:55,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/3126f9b5ef8e45f581eb0f9ad42d00e5, entries=150, sequenceid=176, filesize=11.9 K 2024-11-25T03:13:55,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 159c9b03400b7fa4e53f5245dc0f2c9f in 553ms, sequenceid=176, compaction requested=false 2024-11-25T03:13:55,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:56,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T03:13:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:56,023 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T03:13:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:56,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/bf146e555f114543b017cd9d8e04a6fb is 50, key is test_row_0/A:col10/1732504435475/Put/seqid=0 2024-11-25T03:13:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:13:56,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741879_1055 (size=12151) 2024-11-25T03:13:56,038 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/bf146e555f114543b017cd9d8e04a6fb 2024-11-25T03:13:56,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8 is 50, key is test_row_0/B:col10/1732504435475/Put/seqid=0 2024-11-25T03:13:56,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741880_1056 (size=12151) 2024-11-25T03:13:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:56,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504496125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504496129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504496131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504496134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504496134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504496240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504496240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504496241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504496250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504496250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,465 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8 2024-11-25T03:13:56,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504496460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504496460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504496460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504496460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504496461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c600bbbc0f4a426e9219510e5eb735d2 is 50, key is test_row_0/C:col10/1732504435475/Put/seqid=0 2024-11-25T03:13:56,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741881_1057 (size=12151) 2024-11-25T03:13:56,491 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c600bbbc0f4a426e9219510e5eb735d2 2024-11-25T03:13:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/bf146e555f114543b017cd9d8e04a6fb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bf146e555f114543b017cd9d8e04a6fb 2024-11-25T03:13:56,508 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bf146e555f114543b017cd9d8e04a6fb, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T03:13:56,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8 2024-11-25T03:13:56,517 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T03:13:56,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c600bbbc0f4a426e9219510e5eb735d2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c600bbbc0f4a426e9219510e5eb735d2 2024-11-25T03:13:56,533 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c600bbbc0f4a426e9219510e5eb735d2, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T03:13:56,535 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 159c9b03400b7fa4e53f5245dc0f2c9f in 511ms, sequenceid=193, compaction requested=true 2024-11-25T03:13:56,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:56,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:56,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-25T03:13:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-25T03:13:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:13:56,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-25T03:13:56,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1370 sec 2024-11-25T03:13:56,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1510 sec 2024-11-25T03:13:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:56,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:13:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:56,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:56,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/f7967477466b497d98a04a828206dafd is 50, key is test_row_0/A:col10/1732504436772/Put/seqid=0 2024-11-25T03:13:56,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741882_1058 (size=12151) 2024-11-25T03:13:56,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504496834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504496834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504496836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504496837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504496844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504496950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504496950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504496950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504496951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504496960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504497160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504497160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504497160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504497160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504497170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/f7967477466b497d98a04a828206dafd 2024-11-25T03:13:57,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/44987291156c4a92a52a0b760cbca394 is 50, key is test_row_0/B:col10/1732504436772/Put/seqid=0 2024-11-25T03:13:57,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741883_1059 (size=12151) 2024-11-25T03:13:57,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504497470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504497470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504497470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504497470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504497480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:13:57,550 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:13:57,550 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:13:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-25T03:13:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:13:57,553 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:13:57,554 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:13:57,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:13:57,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/44987291156c4a92a52a0b760cbca394 2024-11-25T03:13:57,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/fcf40bb0e1454fc78438c0acbe51a4bd is 50, key is test_row_0/C:col10/1732504436772/Put/seqid=0 2024-11-25T03:13:57,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741884_1060 (size=12151) 2024-11-25T03:13:57,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/fcf40bb0e1454fc78438c0acbe51a4bd 2024-11-25T03:13:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/f7967477466b497d98a04a828206dafd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f7967477466b497d98a04a828206dafd 2024-11-25T03:13:57,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f7967477466b497d98a04a828206dafd, entries=150, sequenceid=212, filesize=11.9 K 2024-11-25T03:13:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/44987291156c4a92a52a0b760cbca394 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/44987291156c4a92a52a0b760cbca394 2024-11-25T03:13:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:13:57,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/44987291156c4a92a52a0b760cbca394, entries=150, sequenceid=212, filesize=11.9 K 2024-11-25T03:13:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/fcf40bb0e1454fc78438c0acbe51a4bd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/fcf40bb0e1454fc78438c0acbe51a4bd 2024-11-25T03:13:57,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/fcf40bb0e1454fc78438c0acbe51a4bd, entries=150, sequenceid=212, filesize=11.9 K 2024-11-25T03:13:57,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 159c9b03400b7fa4e53f5245dc0f2c9f in 902ms, sequenceid=212, compaction requested=true 2024-11-25T03:13:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:57,677 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:13:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:13:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:57,678 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:57,679 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:13:57,679 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:13:57,679 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:57,679 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e9de676ee33a474a932919299e7233c1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6e91bd6512b45dc88267fefcce96a3d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bf146e555f114543b017cd9d8e04a6fb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f7967477466b497d98a04a828206dafd] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=52.5 K 2024-11-25T03:13:57,680 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e9de676ee33a474a932919299e7233c1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732504433595 2024-11-25T03:13:57,681 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e91bd6512b45dc88267fefcce96a3d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732504434789 2024-11-25T03:13:57,681 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:13:57,682 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:13:57,682 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:57,682 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/457a2ab9b81d41ae9f9adc080d69f433, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/107af241e0064457959f9e0198ead7a7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/44987291156c4a92a52a0b760cbca394] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=47.9 K 2024-11-25T03:13:57,682 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting bf146e555f114543b017cd9d8e04a6fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732504435456 2024-11-25T03:13:57,683 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 457a2ab9b81d41ae9f9adc080d69f433, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732504433595 2024-11-25T03:13:57,683 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f7967477466b497d98a04a828206dafd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732504436128 2024-11-25T03:13:57,685 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 107af241e0064457959f9e0198ead7a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732504434791 2024-11-25T03:13:57,686 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c0eaf00c7b747e9a46a1f8fb6e9e4a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732504435456 2024-11-25T03:13:57,687 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44987291156c4a92a52a0b760cbca394, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732504436128 2024-11-25T03:13:57,701 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:57,702 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/bed41a0f6b8b4d2e8ab057d6f07af6ca is 50, key is test_row_0/A:col10/1732504436772/Put/seqid=0 2024-11-25T03:13:57,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T03:13:57,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:57,708 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:57,708 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T03:13:57,708 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/40f3d48f430d454ca1a6c1138f17c94b is 50, key is test_row_0/B:col10/1732504436772/Put/seqid=0 2024-11-25T03:13:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/de6dd82725d246dfa97349b6ed2b4597 is 50, key is test_row_0/A:col10/1732504436835/Put/seqid=0 2024-11-25T03:13:57,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741886_1062 (size=12697) 2024-11-25T03:13:57,751 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/40f3d48f430d454ca1a6c1138f17c94b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/40f3d48f430d454ca1a6c1138f17c94b 2024-11-25T03:13:57,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741885_1061 (size=12697) 2024-11-25T03:13:57,769 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/bed41a0f6b8b4d2e8ab057d6f07af6ca as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bed41a0f6b8b4d2e8ab057d6f07af6ca 2024-11-25T03:13:57,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741887_1063 (size=12151) 2024-11-25T03:13:57,780 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into 40f3d48f430d454ca1a6c1138f17c94b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:57,781 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:57,781 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=12, startTime=1732504437678; duration=0sec 2024-11-25T03:13:57,781 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into bed41a0f6b8b4d2e8ab057d6f07af6ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:57,781 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:57,782 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=12, startTime=1732504437677; duration=0sec 2024-11-25T03:13:57,782 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:57,782 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:13:57,782 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:13:57,782 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:57,782 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:13:57,788 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:13:57,788 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:13:57,789 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:57,789 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/5369c9fb9e164a1d86324fc83e90e001, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/3126f9b5ef8e45f581eb0f9ad42d00e5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c600bbbc0f4a426e9219510e5eb735d2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/fcf40bb0e1454fc78438c0acbe51a4bd] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=47.9 K 2024-11-25T03:13:57,790 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5369c9fb9e164a1d86324fc83e90e001, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732504433595 2024-11-25T03:13:57,791 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3126f9b5ef8e45f581eb0f9ad42d00e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732504434791 2024-11-25T03:13:57,792 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c600bbbc0f4a426e9219510e5eb735d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732504435456 2024-11-25T03:13:57,792 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf40bb0e1454fc78438c0acbe51a4bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732504436128 2024-11-25T03:13:57,811 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:57,812 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c0ae31c94e4542b392c4293e2e6fa5a0 is 50, key is test_row_0/C:col10/1732504436772/Put/seqid=0 2024-11-25T03:13:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741888_1064 (size=12697) 2024-11-25T03:13:57,839 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c0ae31c94e4542b392c4293e2e6fa5a0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c0ae31c94e4542b392c4293e2e6fa5a0 2024-11-25T03:13:57,853 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into c0ae31c94e4542b392c4293e2e6fa5a0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:57,853 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:57,853 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=12, startTime=1732504437678; duration=0sec 2024-11-25T03:13:57,854 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:57,854 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:13:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:13:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:57,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504498005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504498012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504498014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504498015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504498017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504498121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504498121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504498121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504498130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504498131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/de6dd82725d246dfa97349b6ed2b4597 2024-11-25T03:13:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:13:58,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/31bd35bec7e145b1a6eefb2f1a981981 is 50, key is test_row_0/B:col10/1732504436835/Put/seqid=0 2024-11-25T03:13:58,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741889_1065 (size=12151) 2024-11-25T03:13:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504498330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504498330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504498331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504498340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504498340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,596 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/31bd35bec7e145b1a6eefb2f1a981981 2024-11-25T03:13:58,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c8391bcebce74f21b6c8b08fb11afbd3 is 50, key is test_row_0/C:col10/1732504436835/Put/seqid=0 2024-11-25T03:13:58,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741890_1066 (size=12151) 2024-11-25T03:13:58,615 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c8391bcebce74f21b6c8b08fb11afbd3 2024-11-25T03:13:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/de6dd82725d246dfa97349b6ed2b4597 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/de6dd82725d246dfa97349b6ed2b4597 2024-11-25T03:13:58,630 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/de6dd82725d246dfa97349b6ed2b4597, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T03:13:58,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/31bd35bec7e145b1a6eefb2f1a981981 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/31bd35bec7e145b1a6eefb2f1a981981 2024-11-25T03:13:58,639 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/31bd35bec7e145b1a6eefb2f1a981981, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T03:13:58,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/c8391bcebce74f21b6c8b08fb11afbd3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8391bcebce74f21b6c8b08fb11afbd3 2024-11-25T03:13:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504498640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504498640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504498640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,649 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8391bcebce74f21b6c8b08fb11afbd3, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T03:13:58,650 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 159c9b03400b7fa4e53f5245dc0f2c9f in 942ms, sequenceid=225, compaction requested=false 2024-11-25T03:13:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:58,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-25T03:13:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-25T03:13:58,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T03:13:58,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0990 sec 2024-11-25T03:13:58,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1110 sec 2024-11-25T03:13:58,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-25T03:13:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:13:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:13:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:13:58,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/735d7041b64a472583540642e0e50eb0 is 50, key is test_row_0/A:col10/1732504438013/Put/seqid=0 2024-11-25T03:13:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:13:58,690 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:13:58,691 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:13:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-25T03:13:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:13:58,695 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:13:58,697 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:13:58,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:13:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504498697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504498699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741891_1067 (size=14541) 2024-11-25T03:13:58,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/735d7041b64a472583540642e0e50eb0 2024-11-25T03:13:58,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ffbebe6ea29341f3b8e9db4b502bb40e is 50, key is test_row_0/B:col10/1732504438013/Put/seqid=0 2024-11-25T03:13:58,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741892_1068 (size=12151) 2024-11-25T03:13:58,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ffbebe6ea29341f3b8e9db4b502bb40e 2024-11-25T03:13:58,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/e5f87a13d6254f04a9213699dd88177c is 50, key is test_row_0/C:col10/1732504438013/Put/seqid=0 2024-11-25T03:13:58,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741893_1069 (size=12151) 2024-11-25T03:13:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:13:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504498810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504498810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:58,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T03:13:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:58,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:59,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T03:13:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:13:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504499020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504499020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T03:13:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504499160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504499161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504499161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/e5f87a13d6254f04a9213699dd88177c 2024-11-25T03:13:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/735d7041b64a472583540642e0e50eb0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/735d7041b64a472583540642e0e50eb0 2024-11-25T03:13:59,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/735d7041b64a472583540642e0e50eb0, entries=200, sequenceid=249, filesize=14.2 K 2024-11-25T03:13:59,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ffbebe6ea29341f3b8e9db4b502bb40e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffbebe6ea29341f3b8e9db4b502bb40e 2024-11-25T03:13:59,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffbebe6ea29341f3b8e9db4b502bb40e, entries=150, sequenceid=249, filesize=11.9 K 2024-11-25T03:13:59,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/e5f87a13d6254f04a9213699dd88177c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e5f87a13d6254f04a9213699dd88177c 2024-11-25T03:13:59,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e5f87a13d6254f04a9213699dd88177c, entries=150, sequenceid=249, filesize=11.9 K 2024-11-25T03:13:59,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 159c9b03400b7fa4e53f5245dc0f2c9f in 552ms, sequenceid=249, compaction requested=true 2024-11-25T03:13:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:13:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:59,218 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:13:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:59,218 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:13:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:59,219 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:59,220 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:13:59,220 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,220 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/40f3d48f430d454ca1a6c1138f17c94b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/31bd35bec7e145b1a6eefb2f1a981981, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffbebe6ea29341f3b8e9db4b502bb40e] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.1 K 2024-11-25T03:13:59,220 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:59,220 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:13:59,220 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,221 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bed41a0f6b8b4d2e8ab057d6f07af6ca, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/de6dd82725d246dfa97349b6ed2b4597, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/735d7041b64a472583540642e0e50eb0] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=38.5 K 2024-11-25T03:13:59,221 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f3d48f430d454ca1a6c1138f17c94b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732504436128 2024-11-25T03:13:59,221 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 31bd35bec7e145b1a6eefb2f1a981981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732504436787 2024-11-25T03:13:59,221 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting bed41a0f6b8b4d2e8ab057d6f07af6ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732504436128 2024-11-25T03:13:59,222 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbebe6ea29341f3b8e9db4b502bb40e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732504438012 2024-11-25T03:13:59,222 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6dd82725d246dfa97349b6ed2b4597, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732504436787 2024-11-25T03:13:59,223 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 735d7041b64a472583540642e0e50eb0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732504438002 2024-11-25T03:13:59,241 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:59,242 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/9c320180704747d6a1f7688fbec522d4 is 50, key is test_row_0/B:col10/1732504438013/Put/seqid=0 2024-11-25T03:13:59,246 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:59,247 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/371c9153b08f4a759bc3314975766bbc is 50, key is test_row_0/A:col10/1732504438013/Put/seqid=0 2024-11-25T03:13:59,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741894_1070 (size=12799) 2024-11-25T03:13:59,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741895_1071 (size=12799) 2024-11-25T03:13:59,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T03:13:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,318 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:13:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:13:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:13:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:13:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:13:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:13:59,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/33efec35d24f4423a5760d3b026edfd9 is 50, key is test_row_1/A:col10/1732504438685/Put/seqid=0 2024-11-25T03:13:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:13:59,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:13:59,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741896_1072 (size=9857) 2024-11-25T03:13:59,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504439365, max timestamp=1732504439390Num uniques 859; 2024-11-25T03:13:59,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504439365, max timestamp=1732504439390Num uniques 859; 2024-11-25T03:13:59,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:13:59,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504439365, max timestamp=1732504439390Num uniques 859; 2024-11-25T03:13:59,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:13:59,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:13:59,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504499463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504499470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504499580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504499580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,712 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/9c320180704747d6a1f7688fbec522d4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9c320180704747d6a1f7688fbec522d4 2024-11-25T03:13:59,722 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/371c9153b08f4a759bc3314975766bbc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/371c9153b08f4a759bc3314975766bbc 2024-11-25T03:13:59,729 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into 9c320180704747d6a1f7688fbec522d4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:59,729 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:59,729 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504439218; duration=0sec 2024-11-25T03:13:59,730 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:13:59,730 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:13:59,730 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:13:59,734 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:13:59,734 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:13:59,734 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:13:59,734 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c0ae31c94e4542b392c4293e2e6fa5a0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8391bcebce74f21b6c8b08fb11afbd3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e5f87a13d6254f04a9213699dd88177c] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.1 K 2024-11-25T03:13:59,735 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting c0ae31c94e4542b392c4293e2e6fa5a0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732504436128 2024-11-25T03:13:59,737 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting c8391bcebce74f21b6c8b08fb11afbd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732504436787 2024-11-25T03:13:59,737 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into 371c9153b08f4a759bc3314975766bbc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:59,737 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:59,737 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504439218; duration=0sec 2024-11-25T03:13:59,738 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:59,738 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:13:59,738 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e5f87a13d6254f04a9213699dd88177c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732504438012 2024-11-25T03:13:59,755 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/33efec35d24f4423a5760d3b026edfd9 2024-11-25T03:13:59,755 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:13:59,757 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/9c7f0d2c590b4827b4df71e22b0dbd31 is 50, key is test_row_0/C:col10/1732504438013/Put/seqid=0 2024-11-25T03:13:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/83738761c5a74d2281cdc60be190cbaa is 50, key is test_row_1/B:col10/1732504438685/Put/seqid=0 2024-11-25T03:13:59,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741897_1073 (size=12799) 2024-11-25T03:13:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741898_1074 (size=9857) 2024-11-25T03:13:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504499791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:13:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504499791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:13:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:13:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:13:59,800 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/83738761c5a74d2281cdc60be190cbaa 2024-11-25T03:13:59,811 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/9c7f0d2c590b4827b4df71e22b0dbd31 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/9c7f0d2c590b4827b4df71e22b0dbd31 2024-11-25T03:13:59,830 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into 9c7f0d2c590b4827b4df71e22b0dbd31(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:13:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:13:59,830 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:13:59,830 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504439218; duration=0sec 2024-11-25T03:13:59,830 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:13:59,830 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:13:59,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8830750fa2fe4a14a01442c2a8d5acb3 is 50, key is test_row_1/C:col10/1732504438685/Put/seqid=0 2024-11-25T03:13:59,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741899_1075 (size=9857) 2024-11-25T03:14:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504500100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504500100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504500170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,176 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504500180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504500180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,185 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,186 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:00,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,249 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8830750fa2fe4a14a01442c2a8d5acb3 2024-11-25T03:14:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/33efec35d24f4423a5760d3b026edfd9 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/33efec35d24f4423a5760d3b026edfd9 2024-11-25T03:14:00,262 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/33efec35d24f4423a5760d3b026edfd9, entries=100, sequenceid=260, filesize=9.6 K 2024-11-25T03:14:00,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/83738761c5a74d2281cdc60be190cbaa as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/83738761c5a74d2281cdc60be190cbaa 2024-11-25T03:14:00,270 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/83738761c5a74d2281cdc60be190cbaa, entries=100, sequenceid=260, filesize=9.6 K 2024-11-25T03:14:00,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8830750fa2fe4a14a01442c2a8d5acb3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8830750fa2fe4a14a01442c2a8d5acb3 2024-11-25T03:14:00,277 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8830750fa2fe4a14a01442c2a8d5acb3, entries=100, sequenceid=260, filesize=9.6 K 2024-11-25T03:14:00,279 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 159c9b03400b7fa4e53f5245dc0f2c9f in 961ms, sequenceid=260, compaction requested=false 2024-11-25T03:14:00,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:00,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:00,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-25T03:14:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-25T03:14:00,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T03:14:00,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5850 sec 2024-11-25T03:14:00,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5950 sec 2024-11-25T03:14:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:00,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-25T03:14:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:00,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:00,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/e426233b5e2a47139a506c004d0819e9 is 50, key is test_row_0/A:col10/1732504440611/Put/seqid=0 2024-11-25T03:14:00,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741900_1076 (size=14741) 2024-11-25T03:14:00,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504500632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/e426233b5e2a47139a506c004d0819e9 2024-11-25T03:14:00,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504500634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/f39c0a92f1164c368b5fd2787da5e508 is 50, key is test_row_0/B:col10/1732504440611/Put/seqid=0 2024-11-25T03:14:00,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741901_1077 (size=12301) 2024-11-25T03:14:00,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504500740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504500740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:14:00,841 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:00,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-25T03:14:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:14:00,844 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:00,845 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:00,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:14:00,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504500951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504500951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:00,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T03:14:00,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/f39c0a92f1164c368b5fd2787da5e508 2024-11-25T03:14:01,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1b648de9a2944d61a9758397f9dbe1b3 is 50, key is test_row_0/C:col10/1732504440611/Put/seqid=0 2024-11-25T03:14:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741902_1078 (size=12301) 2024-11-25T03:14:01,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T03:14:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:01,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:14:01,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504501260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:01,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504501260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:01,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T03:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T03:14:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:14:01,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1b648de9a2944d61a9758397f9dbe1b3 2024-11-25T03:14:01,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/e426233b5e2a47139a506c004d0819e9 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e426233b5e2a47139a506c004d0819e9 2024-11-25T03:14:01,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e426233b5e2a47139a506c004d0819e9, entries=200, sequenceid=298, filesize=14.4 K 2024-11-25T03:14:01,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/f39c0a92f1164c368b5fd2787da5e508 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f39c0a92f1164c368b5fd2787da5e508 2024-11-25T03:14:01,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f39c0a92f1164c368b5fd2787da5e508, entries=150, sequenceid=298, filesize=12.0 K 2024-11-25T03:14:01,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/1b648de9a2944d61a9758397f9dbe1b3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b648de9a2944d61a9758397f9dbe1b3 2024-11-25T03:14:01,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b648de9a2944d61a9758397f9dbe1b3, entries=150, sequenceid=298, filesize=12.0 K 2024-11-25T03:14:01,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 159c9b03400b7fa4e53f5245dc0f2c9f in 954ms, sequenceid=298, compaction requested=true 2024-11-25T03:14:01,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:01,568 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:01,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:01,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:01,569 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:01,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:01,570 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:01,570 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:14:01,570 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,570 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/371c9153b08f4a759bc3314975766bbc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/33efec35d24f4423a5760d3b026edfd9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e426233b5e2a47139a506c004d0819e9] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.5 K 2024-11-25T03:14:01,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:01,571 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:01,571 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:14:01,571 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,571 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9c320180704747d6a1f7688fbec522d4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/83738761c5a74d2281cdc60be190cbaa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f39c0a92f1164c368b5fd2787da5e508] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=34.1 K 2024-11-25T03:14:01,571 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 371c9153b08f4a759bc3314975766bbc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732504438012 2024-11-25T03:14:01,572 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c320180704747d6a1f7688fbec522d4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732504438012 2024-11-25T03:14:01,572 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33efec35d24f4423a5760d3b026edfd9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504438685 2024-11-25T03:14:01,573 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 83738761c5a74d2281cdc60be190cbaa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504438685 2024-11-25T03:14:01,573 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting e426233b5e2a47139a506c004d0819e9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732504439429 2024-11-25T03:14:01,574 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f39c0a92f1164c368b5fd2787da5e508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732504439452 2024-11-25T03:14:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:01,595 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:01,597 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/62fc93e1d07d4c90924ff463ae879609 is 50, key is test_row_0/A:col10/1732504440611/Put/seqid=0 2024-11-25T03:14:01,603 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:01,603 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/d818a6a14ec645069d6dcd5bd78a5a1c is 50, key is test_row_0/B:col10/1732504440611/Put/seqid=0 2024-11-25T03:14:01,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T03:14:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,619 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:14:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:01,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741903_1079 (size=13051) 2024-11-25T03:14:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c6847df1703e49158cbfd97bcfb2d5d7 is 50, key is test_row_0/A:col10/1732504440623/Put/seqid=0 2024-11-25T03:14:01,632 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/62fc93e1d07d4c90924ff463ae879609 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/62fc93e1d07d4c90924ff463ae879609 2024-11-25T03:14:01,643 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into 62fc93e1d07d4c90924ff463ae879609(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:01,643 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:01,643 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504441568; duration=0sec 2024-11-25T03:14:01,644 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:01,644 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:14:01,644 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:01,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741904_1080 (size=13051) 2024-11-25T03:14:01,646 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:01,647 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:14:01,647 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,647 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/9c7f0d2c590b4827b4df71e22b0dbd31, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8830750fa2fe4a14a01442c2a8d5acb3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b648de9a2944d61a9758397f9dbe1b3] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=34.1 K 2024-11-25T03:14:01,648 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7f0d2c590b4827b4df71e22b0dbd31, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732504438012 2024-11-25T03:14:01,648 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8830750fa2fe4a14a01442c2a8d5acb3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504438685 2024-11-25T03:14:01,649 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b648de9a2944d61a9758397f9dbe1b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732504439452 2024-11-25T03:14:01,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741905_1081 (size=12301) 2024-11-25T03:14:01,665 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c6847df1703e49158cbfd97bcfb2d5d7 2024-11-25T03:14:01,678 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:01,679 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/cb4629418db4482bad6bb20fd05042fa is 50, key is test_row_0/C:col10/1732504440611/Put/seqid=0 2024-11-25T03:14:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/5b78720d6df2478cb1f8a37c97361cbd is 50, key is test_row_0/B:col10/1732504440623/Put/seqid=0 2024-11-25T03:14:01,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741906_1082 (size=13051) 2024-11-25T03:14:01,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741907_1083 (size=12301) 2024-11-25T03:14:01,716 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/5b78720d6df2478cb1f8a37c97361cbd 2024-11-25T03:14:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8a3a23131a5e40ea848143e677b05a60 is 50, key is test_row_0/C:col10/1732504440623/Put/seqid=0 2024-11-25T03:14:01,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741908_1084 (size=12301) 2024-11-25T03:14:01,759 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8a3a23131a5e40ea848143e677b05a60 2024-11-25T03:14:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c6847df1703e49158cbfd97bcfb2d5d7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6847df1703e49158cbfd97bcfb2d5d7 2024-11-25T03:14:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:01,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:01,775 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6847df1703e49158cbfd97bcfb2d5d7, entries=150, sequenceid=308, filesize=12.0 K 2024-11-25T03:14:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/5b78720d6df2478cb1f8a37c97361cbd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/5b78720d6df2478cb1f8a37c97361cbd 2024-11-25T03:14:01,796 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/5b78720d6df2478cb1f8a37c97361cbd, entries=150, sequenceid=308, filesize=12.0 K 2024-11-25T03:14:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8a3a23131a5e40ea848143e677b05a60 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8a3a23131a5e40ea848143e677b05a60 2024-11-25T03:14:01,806 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8a3a23131a5e40ea848143e677b05a60, entries=150, sequenceid=308, filesize=12.0 K 2024-11-25T03:14:01,807 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=53.67 KB/54960 for 159c9b03400b7fa4e53f5245dc0f2c9f in 188ms, sequenceid=308, compaction requested=false 2024-11-25T03:14:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-25T03:14:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-25T03:14:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:01,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T03:14:01,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:01,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:01,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:01,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:01,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:01,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:01,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-25T03:14:01,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 966 msec 2024-11-25T03:14:01,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 976 msec 2024-11-25T03:14:01,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4e16b95adf814ff4be8a2e8675bd6852 is 50, key is test_row_0/A:col10/1732504441797/Put/seqid=0 2024-11-25T03:14:01,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741909_1085 (size=22065) 2024-11-25T03:14:01,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4e16b95adf814ff4be8a2e8675bd6852 2024-11-25T03:14:01,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/91a981ebde504427911fd2ec5bcf7e1f is 50, key is test_row_0/B:col10/1732504441797/Put/seqid=0 2024-11-25T03:14:01,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504501880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:01,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741910_1086 (size=12301) 2024-11-25T03:14:01,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/91a981ebde504427911fd2ec5bcf7e1f 2024-11-25T03:14:01,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504501888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:01,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/452e2f3784ff4b969089f145f5dbebb0 is 50, key is test_row_0/C:col10/1732504441797/Put/seqid=0 2024-11-25T03:14:01,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741911_1087 (size=12301) 2024-11-25T03:14:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:14:01,981 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:01,981 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-25T03:14:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:14:01,984 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:01,985 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:01,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:02,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504502000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504502000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,056 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/d818a6a14ec645069d6dcd5bd78a5a1c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d818a6a14ec645069d6dcd5bd78a5a1c 2024-11-25T03:14:02,076 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into d818a6a14ec645069d6dcd5bd78a5a1c(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:02,076 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:02,076 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504441569; duration=0sec 2024-11-25T03:14:02,076 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:02,076 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:14:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:14:02,120 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/cb4629418db4482bad6bb20fd05042fa as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/cb4629418db4482bad6bb20fd05042fa 2024-11-25T03:14:02,127 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into cb4629418db4482bad6bb20fd05042fa(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:02,128 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:02,128 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504441575; duration=0sec 2024-11-25T03:14:02,128 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:02,128 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:14:02,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:14:02,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:02,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504502190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,195 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504502200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,201 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504502210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504502210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,219 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:02,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504502211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:14:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:14:02,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/452e2f3784ff4b969089f145f5dbebb0 2024-11-25T03:14:02,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4e16b95adf814ff4be8a2e8675bd6852 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4e16b95adf814ff4be8a2e8675bd6852 2024-11-25T03:14:02,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4e16b95adf814ff4be8a2e8675bd6852, entries=350, sequenceid=322, filesize=21.5 K 2024-11-25T03:14:02,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/91a981ebde504427911fd2ec5bcf7e1f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/91a981ebde504427911fd2ec5bcf7e1f 2024-11-25T03:14:02,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/91a981ebde504427911fd2ec5bcf7e1f, entries=150, sequenceid=322, filesize=12.0 K 2024-11-25T03:14:02,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/452e2f3784ff4b969089f145f5dbebb0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/452e2f3784ff4b969089f145f5dbebb0 2024-11-25T03:14:02,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/452e2f3784ff4b969089f145f5dbebb0, entries=150, sequenceid=322, filesize=12.0 K 2024-11-25T03:14:02,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 159c9b03400b7fa4e53f5245dc0f2c9f in 553ms, sequenceid=322, compaction requested=true 2024-11-25T03:14:02,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:02,364 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:02,364 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:02,366 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:02,366 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:14:02,366 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,366 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/62fc93e1d07d4c90924ff463ae879609, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6847df1703e49158cbfd97bcfb2d5d7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4e16b95adf814ff4be8a2e8675bd6852] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=46.3 K 2024-11-25T03:14:02,366 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:02,367 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:14:02,367 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,367 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d818a6a14ec645069d6dcd5bd78a5a1c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/5b78720d6df2478cb1f8a37c97361cbd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/91a981ebde504427911fd2ec5bcf7e1f] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.8 K 2024-11-25T03:14:02,367 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62fc93e1d07d4c90924ff463ae879609, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732504439452 2024-11-25T03:14:02,367 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d818a6a14ec645069d6dcd5bd78a5a1c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732504439452 2024-11-25T03:14:02,368 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6847df1703e49158cbfd97bcfb2d5d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732504440614 2024-11-25T03:14:02,369 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b78720d6df2478cb1f8a37c97361cbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732504440614 2024-11-25T03:14:02,371 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e16b95adf814ff4be8a2e8675bd6852, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732504441780 2024-11-25T03:14:02,371 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a981ebde504427911fd2ec5bcf7e1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732504441797 2024-11-25T03:14:02,387 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:02,387 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/96ea9d402c794dcd98e027bc34bf36e8 is 50, key is test_row_0/B:col10/1732504441797/Put/seqid=0 2024-11-25T03:14:02,388 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:02,389 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c5398fd5d6f04dd8abb86dc6ce1e52e8 is 50, key is test_row_0/A:col10/1732504441797/Put/seqid=0 2024-11-25T03:14:02,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741912_1088 (size=13153) 2024-11-25T03:14:02,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741913_1089 (size=13153) 2024-11-25T03:14:02,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:14:02,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,452 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:14:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/55c0e289c196456587738d0efbecb3b8 is 50, key is test_row_0/A:col10/1732504441871/Put/seqid=0 2024-11-25T03:14:02,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741914_1090 (size=12301) 2024-11-25T03:14:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:02,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504502560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504502567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:14:02,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504502680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504502680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,814 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/c5398fd5d6f04dd8abb86dc6ce1e52e8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c5398fd5d6f04dd8abb86dc6ce1e52e8 2024-11-25T03:14:02,821 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into c5398fd5d6f04dd8abb86dc6ce1e52e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:02,821 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:02,821 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504442364; duration=0sec 2024-11-25T03:14:02,821 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:02,821 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:14:02,822 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:02,824 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/96ea9d402c794dcd98e027bc34bf36e8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/96ea9d402c794dcd98e027bc34bf36e8 2024-11-25T03:14:02,825 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:02,825 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:14:02,825 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:02,826 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/cb4629418db4482bad6bb20fd05042fa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8a3a23131a5e40ea848143e677b05a60, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/452e2f3784ff4b969089f145f5dbebb0] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.8 K 2024-11-25T03:14:02,826 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb4629418db4482bad6bb20fd05042fa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732504439452 2024-11-25T03:14:02,827 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3a23131a5e40ea848143e677b05a60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732504440614 2024-11-25T03:14:02,828 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452e2f3784ff4b969089f145f5dbebb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732504441797 2024-11-25T03:14:02,832 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into 96ea9d402c794dcd98e027bc34bf36e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:02,832 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:02,832 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504442364; duration=0sec 2024-11-25T03:14:02,832 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:02,832 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:14:02,839 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:02,840 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/03c576035b964880a142b71f8ce321a3 is 50, key is test_row_0/C:col10/1732504441797/Put/seqid=0 2024-11-25T03:14:02,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741915_1091 (size=13153) 2024-11-25T03:14:02,869 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/55c0e289c196456587738d0efbecb3b8 2024-11-25T03:14:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ed1c7bf82b674a5d804314f38bd347f4 is 50, key is test_row_0/B:col10/1732504441871/Put/seqid=0 2024-11-25T03:14:02,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741916_1092 (size=12301) 2024-11-25T03:14:02,885 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ed1c7bf82b674a5d804314f38bd347f4 2024-11-25T03:14:02,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504502890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504502890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/e437aa3b47594eb59c547b229fe5c873 is 50, key is test_row_0/C:col10/1732504441871/Put/seqid=0 2024-11-25T03:14:02,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741917_1093 (size=12301) 2024-11-25T03:14:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:14:03,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504503200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504503200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:03,270 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/03c576035b964880a142b71f8ce321a3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/03c576035b964880a142b71f8ce321a3 2024-11-25T03:14:03,281 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into 03c576035b964880a142b71f8ce321a3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:03,281 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:03,281 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504442364; duration=0sec 2024-11-25T03:14:03,281 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:03,281 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:14:03,305 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/e437aa3b47594eb59c547b229fe5c873 2024-11-25T03:14:03,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/55c0e289c196456587738d0efbecb3b8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/55c0e289c196456587738d0efbecb3b8 2024-11-25T03:14:03,320 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/55c0e289c196456587738d0efbecb3b8, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T03:14:03,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ed1c7bf82b674a5d804314f38bd347f4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ed1c7bf82b674a5d804314f38bd347f4 2024-11-25T03:14:03,330 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ed1c7bf82b674a5d804314f38bd347f4, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T03:14:03,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/e437aa3b47594eb59c547b229fe5c873 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e437aa3b47594eb59c547b229fe5c873 2024-11-25T03:14:03,345 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e437aa3b47594eb59c547b229fe5c873, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T03:14:03,346 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 159c9b03400b7fa4e53f5245dc0f2c9f in 895ms, sequenceid=343, compaction requested=false 2024-11-25T03:14:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:03,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-25T03:14:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-25T03:14:03,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T03:14:03,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3640 sec 2024-11-25T03:14:03,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3720 sec 2024-11-25T03:14:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:03,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T03:14:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:03,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/71b7880b90e24e008704f2057cd50266 is 50, key is test_row_0/A:col10/1732504443711/Put/seqid=0 2024-11-25T03:14:03,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741918_1094 (size=12301) 2024-11-25T03:14:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504503769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:03,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504503769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:03,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504503880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:03,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504503880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:04,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504504090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504504090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:14:04,131 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:04,132 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-25T03:14:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:14:04,135 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:04,136 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:04,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:04,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/71b7880b90e24e008704f2057cd50266 2024-11-25T03:14:04,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/43ca07b787474a64a3e50955d842f9cd is 50, key is test_row_0/B:col10/1732504443711/Put/seqid=0 2024-11-25T03:14:04,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741919_1095 (size=12301) 2024-11-25T03:14:04,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/43ca07b787474a64a3e50955d842f9cd 2024-11-25T03:14:04,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/21a8c4ecb3e94e3fa8ef4b68750aedb9 is 50, key is test_row_0/C:col10/1732504443711/Put/seqid=0 2024-11-25T03:14:04,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741920_1096 (size=12301) 2024-11-25T03:14:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:14:04,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:14:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:04,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504504399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504504400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:04,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:14:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:14:04,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:14:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:04,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/21a8c4ecb3e94e3fa8ef4b68750aedb9 2024-11-25T03:14:04,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/71b7880b90e24e008704f2057cd50266 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/71b7880b90e24e008704f2057cd50266 2024-11-25T03:14:04,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/71b7880b90e24e008704f2057cd50266, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T03:14:04,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/43ca07b787474a64a3e50955d842f9cd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/43ca07b787474a64a3e50955d842f9cd 2024-11-25T03:14:04,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/43ca07b787474a64a3e50955d842f9cd, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T03:14:04,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/21a8c4ecb3e94e3fa8ef4b68750aedb9 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/21a8c4ecb3e94e3fa8ef4b68750aedb9 2024-11-25T03:14:04,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/21a8c4ecb3e94e3fa8ef4b68750aedb9, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T03:14:04,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 159c9b03400b7fa4e53f5245dc0f2c9f in 933ms, sequenceid=360, compaction requested=true 2024-11-25T03:14:04,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:04,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:04,646 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:04,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:04,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:04,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:04,646 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:04,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:04,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:04,647 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:04,647 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:14:04,647 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,648 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:04,648 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c5398fd5d6f04dd8abb86dc6ce1e52e8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/55c0e289c196456587738d0efbecb3b8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/71b7880b90e24e008704f2057cd50266] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.9 K 2024-11-25T03:14:04,648 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:14:04,648 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,648 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/96ea9d402c794dcd98e027bc34bf36e8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ed1c7bf82b674a5d804314f38bd347f4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/43ca07b787474a64a3e50955d842f9cd] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.9 K 2024-11-25T03:14:04,649 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ea9d402c794dcd98e027bc34bf36e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732504441797 2024-11-25T03:14:04,651 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5398fd5d6f04dd8abb86dc6ce1e52e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732504441797 2024-11-25T03:14:04,651 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ed1c7bf82b674a5d804314f38bd347f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732504441871 2024-11-25T03:14:04,652 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ca07b787474a64a3e50955d842f9cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504442558 2024-11-25T03:14:04,652 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c0e289c196456587738d0efbecb3b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732504441871 2024-11-25T03:14:04,653 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71b7880b90e24e008704f2057cd50266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504442558 2024-11-25T03:14:04,667 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:04,667 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:04,667 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/5ea9edc9cb2b4681a54fbae7775f361e is 50, key is test_row_0/A:col10/1732504443711/Put/seqid=0 2024-11-25T03:14:04,668 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/b6aa34433202446dbcadd9873e343198 is 50, key is test_row_0/B:col10/1732504443711/Put/seqid=0 2024-11-25T03:14:04,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741921_1097 (size=13255) 2024-11-25T03:14:04,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741922_1098 (size=13255) 2024-11-25T03:14:04,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:14:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:04,751 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:14:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:04,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:04,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:04,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:04,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4da472787ada4d629d23d58fc0d7416f is 50, key is test_row_0/A:col10/1732504443767/Put/seqid=0 2024-11-25T03:14:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:14:04,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741923_1099 (size=12301) 2024-11-25T03:14:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:04,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:04,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504504975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:04,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504504976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,095 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/5ea9edc9cb2b4681a54fbae7775f361e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/5ea9edc9cb2b4681a54fbae7775f361e 2024-11-25T03:14:05,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504505090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504505091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,106 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/b6aa34433202446dbcadd9873e343198 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/b6aa34433202446dbcadd9873e343198 2024-11-25T03:14:05,117 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into b6aa34433202446dbcadd9873e343198(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:05,118 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:05,118 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504444646; duration=0sec 2024-11-25T03:14:05,118 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:05,118 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:14:05,118 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:05,120 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:05,120 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:14:05,121 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:05,121 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/03c576035b964880a142b71f8ce321a3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e437aa3b47594eb59c547b229fe5c873, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/21a8c4ecb3e94e3fa8ef4b68750aedb9] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=36.9 K 2024-11-25T03:14:05,122 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c576035b964880a142b71f8ce321a3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732504441797 2024-11-25T03:14:05,122 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e437aa3b47594eb59c547b229fe5c873, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732504441871 2024-11-25T03:14:05,124 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a8c4ecb3e94e3fa8ef4b68750aedb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504442558 2024-11-25T03:14:05,128 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into 5ea9edc9cb2b4681a54fbae7775f361e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:05,128 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:05,128 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504444646; duration=0sec 2024-11-25T03:14:05,128 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:05,129 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:14:05,144 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:05,145 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/38818c3fb71640e2bbd89983d95b5bb0 is 50, key is test_row_0/C:col10/1732504443711/Put/seqid=0 2024-11-25T03:14:05,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741924_1100 (size=13255) 2024-11-25T03:14:05,163 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4da472787ada4d629d23d58fc0d7416f 2024-11-25T03:14:05,172 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/38818c3fb71640e2bbd89983d95b5bb0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/38818c3fb71640e2bbd89983d95b5bb0 2024-11-25T03:14:05,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/90a036bfe6764cec9b3aa26f36386230 is 50, key is test_row_0/B:col10/1732504443767/Put/seqid=0 2024-11-25T03:14:05,185 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into 38818c3fb71640e2bbd89983d95b5bb0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:05,185 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:05,186 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504444646; duration=0sec 2024-11-25T03:14:05,186 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:05,186 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:14:05,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741925_1101 (size=12301) 2024-11-25T03:14:05,200 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/90a036bfe6764cec9b3aa26f36386230 2024-11-25T03:14:05,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/22ff8e1cbb7c4a90a52af988a5f67d94 is 50, key is test_row_0/C:col10/1732504443767/Put/seqid=0 2024-11-25T03:14:05,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741926_1102 (size=12301) 2024-11-25T03:14:05,239 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/22ff8e1cbb7c4a90a52af988a5f67d94 2024-11-25T03:14:05,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4da472787ada4d629d23d58fc0d7416f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4da472787ada4d629d23d58fc0d7416f 2024-11-25T03:14:05,258 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4da472787ada4d629d23d58fc0d7416f, entries=150, sequenceid=379, filesize=12.0 K 2024-11-25T03:14:05,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/90a036bfe6764cec9b3aa26f36386230 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/90a036bfe6764cec9b3aa26f36386230 2024-11-25T03:14:05,266 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/90a036bfe6764cec9b3aa26f36386230, entries=150, sequenceid=379, filesize=12.0 K 2024-11-25T03:14:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/22ff8e1cbb7c4a90a52af988a5f67d94 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/22ff8e1cbb7c4a90a52af988a5f67d94 2024-11-25T03:14:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:14:05,275 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/22ff8e1cbb7c4a90a52af988a5f67d94, entries=150, sequenceid=379, filesize=12.0 K 2024-11-25T03:14:05,277 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 159c9b03400b7fa4e53f5245dc0f2c9f in 526ms, sequenceid=379, compaction requested=false 2024-11-25T03:14:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-25T03:14:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-25T03:14:05,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T03:14:05,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1430 sec 2024-11-25T03:14:05,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1510 sec 2024-11-25T03:14:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:05,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:14:05,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:05,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:05,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:05,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:05,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:05,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:05,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/60c677126a59488196102e7e87cb28d6 is 50, key is test_row_0/A:col10/1732504445312/Put/seqid=0 2024-11-25T03:14:05,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741927_1103 (size=12301) 2024-11-25T03:14:05,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/60c677126a59488196102e7e87cb28d6 2024-11-25T03:14:05,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/f69c2a970112464a8cbda95b0a1b5f20 is 50, key is test_row_0/B:col10/1732504445312/Put/seqid=0 2024-11-25T03:14:05,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504505370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504505372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741928_1104 (size=12301) 2024-11-25T03:14:05,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/f69c2a970112464a8cbda95b0a1b5f20 2024-11-25T03:14:05,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/757d1a99979e4255a8f33f62825fe1cb is 50, key is test_row_0/C:col10/1732504445312/Put/seqid=0 2024-11-25T03:14:05,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741929_1105 (size=12301) 2024-11-25T03:14:05,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504505482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504505482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504505690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504505700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:05,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/757d1a99979e4255a8f33f62825fe1cb 2024-11-25T03:14:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/60c677126a59488196102e7e87cb28d6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/60c677126a59488196102e7e87cb28d6 2024-11-25T03:14:05,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/60c677126a59488196102e7e87cb28d6, entries=150, sequenceid=398, filesize=12.0 K 2024-11-25T03:14:05,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/f69c2a970112464a8cbda95b0a1b5f20 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f69c2a970112464a8cbda95b0a1b5f20 2024-11-25T03:14:05,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f69c2a970112464a8cbda95b0a1b5f20, entries=150, sequenceid=398, filesize=12.0 K 2024-11-25T03:14:05,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/757d1a99979e4255a8f33f62825fe1cb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/757d1a99979e4255a8f33f62825fe1cb 2024-11-25T03:14:05,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/757d1a99979e4255a8f33f62825fe1cb, entries=150, sequenceid=398, filesize=12.0 K 2024-11-25T03:14:05,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 159c9b03400b7fa4e53f5245dc0f2c9f in 529ms, sequenceid=398, compaction requested=true 2024-11-25T03:14:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:05,848 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:05,848 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:05,849 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:05,849 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:05,850 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:14:05,850 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:14:05,850 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:05,850 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:05,850 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/5ea9edc9cb2b4681a54fbae7775f361e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4da472787ada4d629d23d58fc0d7416f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/60c677126a59488196102e7e87cb28d6] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=37.0 K 2024-11-25T03:14:05,850 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/b6aa34433202446dbcadd9873e343198, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/90a036bfe6764cec9b3aa26f36386230, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f69c2a970112464a8cbda95b0a1b5f20] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=37.0 K 2024-11-25T03:14:05,851 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea9edc9cb2b4681a54fbae7775f361e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504442558 2024-11-25T03:14:05,851 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b6aa34433202446dbcadd9873e343198, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504442558 2024-11-25T03:14:05,851 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a036bfe6764cec9b3aa26f36386230, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732504443764 2024-11-25T03:14:05,851 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4da472787ada4d629d23d58fc0d7416f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732504443764 2024-11-25T03:14:05,852 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f69c2a970112464a8cbda95b0a1b5f20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732504444967 2024-11-25T03:14:05,852 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c677126a59488196102e7e87cb28d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732504444967 2024-11-25T03:14:05,861 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:05,862 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/a0392edbac7b40f0be2e2c3a265b5cdb is 50, key is test_row_0/B:col10/1732504445312/Put/seqid=0 2024-11-25T03:14:05,862 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:05,863 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/d331b31bed1c4fb6b8e83c93cdbaa047 is 50, key is test_row_0/A:col10/1732504445312/Put/seqid=0 2024-11-25T03:14:05,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741930_1106 (size=13357) 2024-11-25T03:14:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741931_1107 (size=13357) 2024-11-25T03:14:05,885 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/d331b31bed1c4fb6b8e83c93cdbaa047 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d331b31bed1c4fb6b8e83c93cdbaa047 2024-11-25T03:14:05,893 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into d331b31bed1c4fb6b8e83c93cdbaa047(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:05,893 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:05,893 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=13, startTime=1732504445848; duration=0sec 2024-11-25T03:14:05,893 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:05,893 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:14:05,893 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:05,896 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:05,896 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:14:05,896 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:05,897 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/38818c3fb71640e2bbd89983d95b5bb0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/22ff8e1cbb7c4a90a52af988a5f67d94, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/757d1a99979e4255a8f33f62825fe1cb] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=37.0 K 2024-11-25T03:14:05,897 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38818c3fb71640e2bbd89983d95b5bb0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504442558 2024-11-25T03:14:05,900 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ff8e1cbb7c4a90a52af988a5f67d94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732504443764 2024-11-25T03:14:05,900 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 757d1a99979e4255a8f33f62825fe1cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732504444967 2024-11-25T03:14:05,911 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:05,912 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/bb5d72510444456d8c75650098726b6d is 50, key is test_row_0/C:col10/1732504445312/Put/seqid=0 2024-11-25T03:14:05,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741932_1108 (size=13357) 2024-11-25T03:14:05,926 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/bb5d72510444456d8c75650098726b6d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/bb5d72510444456d8c75650098726b6d 2024-11-25T03:14:05,936 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into bb5d72510444456d8c75650098726b6d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:05,936 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:05,936 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=13, startTime=1732504445848; duration=0sec 2024-11-25T03:14:05,937 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:05,937 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:14:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:06,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:14:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:06,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ef04547df30e4b43be061b11139cedda is 50, key is test_row_0/A:col10/1732504445370/Put/seqid=0 2024-11-25T03:14:06,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741933_1109 (size=14741) 2024-11-25T03:14:06,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ef04547df30e4b43be061b11139cedda 2024-11-25T03:14:06,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504506041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504506042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/7ae025de686746b3b72dffa81e7246fd is 50, key is test_row_0/B:col10/1732504445370/Put/seqid=0 2024-11-25T03:14:06,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741934_1110 (size=12301) 2024-11-25T03:14:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504506150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504506150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37274 deadline: 1732504506220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,221 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37286 deadline: 1732504506229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,232 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1732504506240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,243 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 159c9b03400b7fa4e53f5245dc0f2c9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,278 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/a0392edbac7b40f0be2e2c3a265b5cdb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a0392edbac7b40f0be2e2c3a265b5cdb 2024-11-25T03:14:06,285 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into a0392edbac7b40f0be2e2c3a265b5cdb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:06,286 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:06,286 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=13, startTime=1732504445848; duration=0sec 2024-11-25T03:14:06,286 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:06,286 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:14:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:14:06,291 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:06,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-25T03:14:06,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:14:06,294 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:06,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:06,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504506360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504506360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:14:06,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-25T03:14:06,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/7ae025de686746b3b72dffa81e7246fd 2024-11-25T03:14:06,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/32a085f2416e4330adafb239f7ff45fe is 50, key is test_row_0/C:col10/1732504445370/Put/seqid=0 2024-11-25T03:14:06,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741935_1111 (size=12301) 2024-11-25T03:14:06,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-25T03:14:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:14:06,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504506670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504506680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:06,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-25T03:14:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/32a085f2416e4330adafb239f7ff45fe 2024-11-25T03:14:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ef04547df30e4b43be061b11139cedda as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef04547df30e4b43be061b11139cedda 2024-11-25T03:14:06,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef04547df30e4b43be061b11139cedda, entries=200, sequenceid=417, filesize=14.4 K 2024-11-25T03:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/7ae025de686746b3b72dffa81e7246fd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7ae025de686746b3b72dffa81e7246fd 2024-11-25T03:14:06,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7ae025de686746b3b72dffa81e7246fd, entries=150, sequenceid=417, filesize=12.0 K 2024-11-25T03:14:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/32a085f2416e4330adafb239f7ff45fe as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a085f2416e4330adafb239f7ff45fe 2024-11-25T03:14:06,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-25T03:14:06,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:06,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:06,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:06,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a085f2416e4330adafb239f7ff45fe, entries=150, sequenceid=417, filesize=12.0 K 2024-11-25T03:14:06,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 159c9b03400b7fa4e53f5245dc0f2c9f in 912ms, sequenceid=417, compaction requested=false 2024-11-25T03:14:06,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:14:07,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-25T03:14:07,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:07,063 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:14:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:07,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4ed677b749e14459ab4ddf192dfcd254 is 50, key is test_row_0/A:col10/1732504446038/Put/seqid=0 2024-11-25T03:14:07,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741936_1112 (size=12301) 2024-11-25T03:14:07,076 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4ed677b749e14459ab4ddf192dfcd254 2024-11-25T03:14:07,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/c9f09e545a2c4a5587519ad2b8bc8e68 is 50, key is test_row_0/B:col10/1732504446038/Put/seqid=0 2024-11-25T03:14:07,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741937_1113 (size=12301) 2024-11-25T03:14:07,129 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/c9f09e545a2c4a5587519ad2b8bc8e68 2024-11-25T03:14:07,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/32a55848542a483baf0422298da1d918 is 50, key is test_row_0/C:col10/1732504446038/Put/seqid=0 2024-11-25T03:14:07,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741938_1114 (size=12301) 2024-11-25T03:14:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:07,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:07,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504507223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504507225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504507340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504507340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:14:07,558 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/32a55848542a483baf0422298da1d918 2024-11-25T03:14:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504507565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504507567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/4ed677b749e14459ab4ddf192dfcd254 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4ed677b749e14459ab4ddf192dfcd254 2024-11-25T03:14:07,575 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4ed677b749e14459ab4ddf192dfcd254, entries=150, sequenceid=433, filesize=12.0 K 2024-11-25T03:14:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/c9f09e545a2c4a5587519ad2b8bc8e68 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/c9f09e545a2c4a5587519ad2b8bc8e68 2024-11-25T03:14:07,590 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/c9f09e545a2c4a5587519ad2b8bc8e68, entries=150, sequenceid=433, filesize=12.0 K 2024-11-25T03:14:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/32a55848542a483baf0422298da1d918 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a55848542a483baf0422298da1d918 2024-11-25T03:14:07,596 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a55848542a483baf0422298da1d918, entries=150, sequenceid=433, filesize=12.0 K 2024-11-25T03:14:07,598 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 159c9b03400b7fa4e53f5245dc0f2c9f in 535ms, sequenceid=433, compaction requested=true 2024-11-25T03:14:07,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:07,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:07,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-25T03:14:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-25T03:14:07,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-25T03:14:07,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3060 sec 2024-11-25T03:14:07,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.3120 sec 2024-11-25T03:14:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:07,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:14:07,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:07,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:07,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:07,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:07,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:07,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:07,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/cdaad31fe4574a9fa093c032809640ad is 50, key is test_row_0/A:col10/1732504447218/Put/seqid=0 2024-11-25T03:14:07,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741939_1115 (size=17181) 2024-11-25T03:14:07,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/cdaad31fe4574a9fa093c032809640ad 2024-11-25T03:14:07,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ea1aea42e0054a4bb1be5cf36abadaef is 50, key is test_row_0/B:col10/1732504447218/Put/seqid=0 2024-11-25T03:14:07,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741940_1116 (size=12301) 2024-11-25T03:14:07,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ea1aea42e0054a4bb1be5cf36abadaef 2024-11-25T03:14:07,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504507940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8cf9a83b790b4644b571715bc1390dad is 50, key is test_row_0/C:col10/1732504447218/Put/seqid=0 2024-11-25T03:14:07,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504507942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:07,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741941_1117 (size=12301) 2024-11-25T03:14:08,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504508050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504508050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504508260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504508271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8cf9a83b790b4644b571715bc1390dad 2024-11-25T03:14:08,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/cdaad31fe4574a9fa093c032809640ad as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/cdaad31fe4574a9fa093c032809640ad 2024-11-25T03:14:08,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/cdaad31fe4574a9fa093c032809640ad, entries=250, sequenceid=452, filesize=16.8 K 2024-11-25T03:14:08,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/ea1aea42e0054a4bb1be5cf36abadaef as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ea1aea42e0054a4bb1be5cf36abadaef 2024-11-25T03:14:08,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ea1aea42e0054a4bb1be5cf36abadaef, entries=150, sequenceid=452, filesize=12.0 K 2024-11-25T03:14:08,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/8cf9a83b790b4644b571715bc1390dad as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8cf9a83b790b4644b571715bc1390dad 2024-11-25T03:14:08,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8cf9a83b790b4644b571715bc1390dad, entries=150, sequenceid=452, filesize=12.0 K 2024-11-25T03:14:08,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 159c9b03400b7fa4e53f5245dc0f2c9f in 498ms, sequenceid=452, compaction requested=true 2024-11-25T03:14:08,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:08,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:08,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:08,401 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:08,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:08,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:08,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:08,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T03:14:08,403 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:08,406 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:08,406 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:08,406 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/B is initiating minor compaction (all files) 2024-11-25T03:14:08,406 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/A is initiating minor compaction (all files) 2024-11-25T03:14:08,406 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/B in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,406 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/A in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,406 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a0392edbac7b40f0be2e2c3a265b5cdb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7ae025de686746b3b72dffa81e7246fd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/c9f09e545a2c4a5587519ad2b8bc8e68, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ea1aea42e0054a4bb1be5cf36abadaef] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=49.1 K 2024-11-25T03:14:08,406 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d331b31bed1c4fb6b8e83c93cdbaa047, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef04547df30e4b43be061b11139cedda, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4ed677b749e14459ab4ddf192dfcd254, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/cdaad31fe4574a9fa093c032809640ad] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=56.2 K 2024-11-25T03:14:08,408 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0392edbac7b40f0be2e2c3a265b5cdb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732504444967 2024-11-25T03:14:08,408 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d331b31bed1c4fb6b8e83c93cdbaa047, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732504444967 2024-11-25T03:14:08,408 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ae025de686746b3b72dffa81e7246fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732504445355 2024-11-25T03:14:08,408 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ef04547df30e4b43be061b11139cedda, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732504445355 2024-11-25T03:14:08,409 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f09e545a2c4a5587519ad2b8bc8e68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732504446022 2024-11-25T03:14:08,409 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed677b749e14459ab4ddf192dfcd254, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732504446022 2024-11-25T03:14:08,409 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea1aea42e0054a4bb1be5cf36abadaef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732504447218 2024-11-25T03:14:08,410 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting cdaad31fe4574a9fa093c032809640ad, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732504447218 2024-11-25T03:14:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:14:08,440 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:08,441 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:08,444 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-25T03:14:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:14:08,447 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:08,448 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3e34752d37a948c799637aed43e1d569 is 50, key is test_row_0/B:col10/1732504447218/Put/seqid=0 2024-11-25T03:14:08,453 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:08,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:08,454 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#A#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:08,455 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/01311ef1ac264a3183de6b109bb4ebae is 50, key is test_row_0/A:col10/1732504447218/Put/seqid=0 2024-11-25T03:14:08,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741942_1118 (size=13493) 2024-11-25T03:14:08,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741943_1119 (size=13493) 2024-11-25T03:14:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:14:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:08,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T03:14:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:08,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:08,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:08,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:08,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ddcc32f0872e4cb7a0fcdc0e83f09eb9 is 50, key is test_row_0/A:col10/1732504448583/Put/seqid=0 2024-11-25T03:14:08,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-25T03:14:08,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:08,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741944_1120 (size=17181) 2024-11-25T03:14:08,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ddcc32f0872e4cb7a0fcdc0e83f09eb9 2024-11-25T03:14:08,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/9aaf05bcdefd4678ae852d3b1fd383b3 is 50, key is test_row_0/B:col10/1732504448583/Put/seqid=0 2024-11-25T03:14:08,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504508638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504508640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741945_1121 (size=12301) 2024-11-25T03:14:08,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/9aaf05bcdefd4678ae852d3b1fd383b3 2024-11-25T03:14:08,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/f46cd3ad77234a7c900adb127bf840aa is 50, key is test_row_0/C:col10/1732504448583/Put/seqid=0 2024-11-25T03:14:08,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741946_1122 (size=12301) 2024-11-25T03:14:08,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504508750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:14:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504508760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-25T03:14:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:08,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,908 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3e34752d37a948c799637aed43e1d569 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3e34752d37a948c799637aed43e1d569 2024-11-25T03:14:08,914 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/01311ef1ac264a3183de6b109bb4ebae as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/01311ef1ac264a3183de6b109bb4ebae 2024-11-25T03:14:08,920 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/B of 159c9b03400b7fa4e53f5245dc0f2c9f into 3e34752d37a948c799637aed43e1d569(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:08,921 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:08,921 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/B, priority=12, startTime=1732504448401; duration=0sec 2024-11-25T03:14:08,921 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:08,921 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:14:08,921 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:08,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-25T03:14:08,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:08,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:08,926 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:08,927 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 159c9b03400b7fa4e53f5245dc0f2c9f/C is initiating minor compaction (all files) 2024-11-25T03:14:08,927 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 159c9b03400b7fa4e53f5245dc0f2c9f/C in TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:08,927 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/bb5d72510444456d8c75650098726b6d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a085f2416e4330adafb239f7ff45fe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a55848542a483baf0422298da1d918, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8cf9a83b790b4644b571715bc1390dad] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp, totalSize=49.1 K 2024-11-25T03:14:08,927 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5d72510444456d8c75650098726b6d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732504444967 2024-11-25T03:14:08,928 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a085f2416e4330adafb239f7ff45fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732504445355 2024-11-25T03:14:08,929 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a55848542a483baf0422298da1d918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732504446022 2024-11-25T03:14:08,930 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/A of 159c9b03400b7fa4e53f5245dc0f2c9f into 01311ef1ac264a3183de6b109bb4ebae(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:08,930 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:08,930 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/A, priority=12, startTime=1732504448401; duration=0sec 2024-11-25T03:14:08,930 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:08,930 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:14:08,930 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cf9a83b790b4644b571715bc1390dad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732504447218 2024-11-25T03:14:08,942 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 159c9b03400b7fa4e53f5245dc0f2c9f#C#compaction#110 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:08,943 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/b6efb30cf6ff4fa4bc6db7df300b26f8 is 50, key is test_row_0/C:col10/1732504447218/Put/seqid=0 2024-11-25T03:14:08,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732504508960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741947_1123 (size=13493) 2024-11-25T03:14:08,973 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/b6efb30cf6ff4fa4bc6db7df300b26f8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/b6efb30cf6ff4fa4bc6db7df300b26f8 2024-11-25T03:14:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1732504508970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:08,981 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 159c9b03400b7fa4e53f5245dc0f2c9f/C of 159c9b03400b7fa4e53f5245dc0f2c9f into b6efb30cf6ff4fa4bc6db7df300b26f8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:08,981 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:08,982 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., storeName=159c9b03400b7fa4e53f5245dc0f2c9f/C, priority=12, startTime=1732504448401; duration=0sec 2024-11-25T03:14:08,982 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:08,982 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:14:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:14:09,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-25T03:14:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:09,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:09,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/f46cd3ad77234a7c900adb127bf840aa 2024-11-25T03:14:09,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ddcc32f0872e4cb7a0fcdc0e83f09eb9 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ddcc32f0872e4cb7a0fcdc0e83f09eb9 2024-11-25T03:14:09,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ddcc32f0872e4cb7a0fcdc0e83f09eb9, entries=250, sequenceid=467, filesize=16.8 K 2024-11-25T03:14:09,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/9aaf05bcdefd4678ae852d3b1fd383b3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9aaf05bcdefd4678ae852d3b1fd383b3 2024-11-25T03:14:09,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9aaf05bcdefd4678ae852d3b1fd383b3, entries=150, sequenceid=467, filesize=12.0 K 2024-11-25T03:14:09,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/f46cd3ad77234a7c900adb127bf840aa as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f46cd3ad77234a7c900adb127bf840aa 2024-11-25T03:14:09,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f46cd3ad77234a7c900adb127bf840aa, entries=150, sequenceid=467, filesize=12.0 K 2024-11-25T03:14:09,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 159c9b03400b7fa4e53f5245dc0f2c9f in 546ms, sequenceid=467, compaction requested=false 2024-11-25T03:14:09,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:09,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-25T03:14:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:09,232 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:14:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:09,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:09,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:09,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/0aa3afb9821d4429a22e0114e68ab19e is 50, key is test_row_0/A:col10/1732504448637/Put/seqid=0 2024-11-25T03:14:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:09,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. as already flushing 2024-11-25T03:14:09,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741948_1124 (size=12301) 2024-11-25T03:14:09,308 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-25T03:14:09,308 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:09,311 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-25T03:14:09,311 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:09,317 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-25T03:14:09,317 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:09,318 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-25T03:14:09,318 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:09,319 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-25T03:14:09,319 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:09,320 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,320 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,320 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,320 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,321 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,321 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,321 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:09,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:09,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:09,322 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-25T03:14:09,322 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:09,322 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,322 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,322 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:09,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:09,323 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,323 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:09,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:14:09,687 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/0aa3afb9821d4429a22e0114e68ab19e 2024-11-25T03:14:09,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/22e864c1c70542a096bbd90afd0c7a09 is 50, key is test_row_0/B:col10/1732504448637/Put/seqid=0 2024-11-25T03:14:09,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741949_1125 (size=12301) 2024-11-25T03:14:10,104 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/22e864c1c70542a096bbd90afd0c7a09 2024-11-25T03:14:10,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/79a79cd2f16142daaea2f55c5dd211e7 is 50, key is test_row_0/C:col10/1732504448637/Put/seqid=0 2024-11-25T03:14:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741950_1126 (size=12301) 2024-11-25T03:14:10,122 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/79a79cd2f16142daaea2f55c5dd211e7 2024-11-25T03:14:10,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/0aa3afb9821d4429a22e0114e68ab19e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/0aa3afb9821d4429a22e0114e68ab19e 2024-11-25T03:14:10,133 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/0aa3afb9821d4429a22e0114e68ab19e, entries=150, sequenceid=487, filesize=12.0 K 2024-11-25T03:14:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/22e864c1c70542a096bbd90afd0c7a09 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/22e864c1c70542a096bbd90afd0c7a09 2024-11-25T03:14:10,139 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/22e864c1c70542a096bbd90afd0c7a09, entries=150, sequenceid=487, filesize=12.0 K 2024-11-25T03:14:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/79a79cd2f16142daaea2f55c5dd211e7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/79a79cd2f16142daaea2f55c5dd211e7 2024-11-25T03:14:10,145 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/79a79cd2f16142daaea2f55c5dd211e7, entries=150, sequenceid=487, filesize=12.0 K 2024-11-25T03:14:10,146 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=53.67 KB/54960 for 159c9b03400b7fa4e53f5245dc0f2c9f in 914ms, sequenceid=487, compaction requested=true 2024-11-25T03:14:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-25T03:14:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-25T03:14:10,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T03:14:10,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6940 sec 2024-11-25T03:14:10,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.7110 sec 2024-11-25T03:14:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:14:10,591 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:13,186 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:14:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:16,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T03:14:16,292 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-25T03:14:16,292 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:16,292 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:16,292 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:16,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:16,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/1aafd632e0b046d688b81fb23f97cdd3 is 50, key is test_row_0/A:col10/1732504449304/Put/seqid=0 2024-11-25T03:14:16,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741951_1127 (size=12301) 2024-11-25T03:14:16,302 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-25T03:14:16,302 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:16,302 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,302 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:16,341 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-25T03:14:16,341 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:16,341 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,342 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2833 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2801 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1239 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3712 rows 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1247 2024-11-25T03:14:16,342 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3736 rows 2024-11-25T03:14:16,342 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:14:16,342 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:14:16,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:16,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:16,347 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:14:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:14:16,354 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504456354"}]},"ts":"1732504456354"} 2024-11-25T03:14:16,357 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:14:16,357 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:14:16,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:14:16,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, UNASSIGN}] 2024-11-25T03:14:16,363 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, UNASSIGN 2024-11-25T03:14:16,364 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=159c9b03400b7fa4e53f5245dc0f2c9f, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:16,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, UNASSIGN because future has completed 2024-11-25T03:14:16,367 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:14:16,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:14:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:14:16,523 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:16,523 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:14:16,524 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 159c9b03400b7fa4e53f5245dc0f2c9f, disabling compactions & flushes 2024-11-25T03:14:16,524 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:14:16,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/1aafd632e0b046d688b81fb23f97cdd3 2024-11-25T03:14:16,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3232bfbfc94347fcafd0cda51db382bc is 50, key is test_row_0/B:col10/1732504449304/Put/seqid=0 2024-11-25T03:14:16,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741952_1128 (size=12301) 2024-11-25T03:14:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:14:17,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3232bfbfc94347fcafd0cda51db382bc 2024-11-25T03:14:17,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/f40aecfbccb148d587b9248417ef14a5 is 50, key is test_row_0/C:col10/1732504449304/Put/seqid=0 2024-11-25T03:14:17,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741953_1129 (size=12301) 2024-11-25T03:14:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:14:17,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/f40aecfbccb148d587b9248417ef14a5 2024-11-25T03:14:17,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/1aafd632e0b046d688b81fb23f97cdd3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1aafd632e0b046d688b81fb23f97cdd3 2024-11-25T03:14:17,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1aafd632e0b046d688b81fb23f97cdd3, entries=150, sequenceid=499, filesize=12.0 K 2024-11-25T03:14:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/3232bfbfc94347fcafd0cda51db382bc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3232bfbfc94347fcafd0cda51db382bc 2024-11-25T03:14:17,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3232bfbfc94347fcafd0cda51db382bc, entries=150, sequenceid=499, filesize=12.0 K 2024-11-25T03:14:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/f40aecfbccb148d587b9248417ef14a5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f40aecfbccb148d587b9248417ef14a5 2024-11-25T03:14:17,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f40aecfbccb148d587b9248417ef14a5, entries=150, sequenceid=499, filesize=12.0 K 2024-11-25T03:14:17,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for 159c9b03400b7fa4e53f5245dc0f2c9f in 1257ms, sequenceid=499, compaction requested=true 2024-11-25T03:14:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 159c9b03400b7fa4e53f5245dc0f2c9f: 2024-11-25T03:14:17,548 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:17,548 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:17,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. after waiting 0 ms 2024-11-25T03:14:17,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:17,549 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. because compaction request was cancelled 2024-11-25T03:14:17,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 159c9b03400b7fa4e53f5245dc0f2c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:17,549 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:A 2024-11-25T03:14:17,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:17,549 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. because compaction request was cancelled 2024-11-25T03:14:17,549 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. because compaction request was cancelled 2024-11-25T03:14:17,549 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:B 2024-11-25T03:14:17,549 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 159c9b03400b7fa4e53f5245dc0f2c9f:C 2024-11-25T03:14:17,549 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 159c9b03400b7fa4e53f5245dc0f2c9f 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=A 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=B 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 159c9b03400b7fa4e53f5245dc0f2c9f, store=C 2024-11-25T03:14:17,549 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:17,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ac096d5ac2ee49c2a91475c03ef2c7f6 is 50, key is test_row_0/A:col10/1732504456340/Put/seqid=0 2024-11-25T03:14:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741954_1130 (size=9857) 2024-11-25T03:14:17,958 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ac096d5ac2ee49c2a91475c03ef2c7f6 2024-11-25T03:14:17,965 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/d718fdb839a447f19943241306fc170f is 50, key is test_row_0/B:col10/1732504456340/Put/seqid=0 2024-11-25T03:14:17,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741955_1131 (size=9857) 2024-11-25T03:14:18,370 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/d718fdb839a447f19943241306fc170f 2024-11-25T03:14:18,378 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/b9f2001e0c994efb9a1d710610bf2a71 is 50, key is test_row_0/C:col10/1732504456340/Put/seqid=0 2024-11-25T03:14:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741956_1132 (size=9857) 2024-11-25T03:14:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:14:18,782 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/b9f2001e0c994efb9a1d710610bf2a71 2024-11-25T03:14:18,787 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/A/ac096d5ac2ee49c2a91475c03ef2c7f6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ac096d5ac2ee49c2a91475c03ef2c7f6 2024-11-25T03:14:18,791 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ac096d5ac2ee49c2a91475c03ef2c7f6, entries=100, sequenceid=504, filesize=9.6 K 2024-11-25T03:14:18,792 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/B/d718fdb839a447f19943241306fc170f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d718fdb839a447f19943241306fc170f 2024-11-25T03:14:18,795 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d718fdb839a447f19943241306fc170f, entries=100, sequenceid=504, filesize=9.6 K 2024-11-25T03:14:18,796 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/.tmp/C/b9f2001e0c994efb9a1d710610bf2a71 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/b9f2001e0c994efb9a1d710610bf2a71 2024-11-25T03:14:18,799 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/b9f2001e0c994efb9a1d710610bf2a71, entries=100, sequenceid=504, filesize=9.6 K 2024-11-25T03:14:18,800 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 159c9b03400b7fa4e53f5245dc0f2c9f in 1251ms, sequenceid=504, compaction requested=true 2024-11-25T03:14:18,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1657a4c67ddc4c51836161bc0d80804c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/211b1461f4de41e9b6e569d1d745b508, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f3ae286bdabf4954a1c1518481ec511a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/a9a52a3cde8442a0a0ce2066718c9e96, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4611b49c14cc4873ab00342b7b61cc03, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/47f83ba0578b4f1c83093dd90fb1ca68, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/dd2123c81f4b4d77ae908317874c237a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/fd160a6838aa4e64a6881b80a72a47ee, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/8462d576bb324b4cb34ad324b77d829a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d850b17fbbd547aaad73e06e87804169, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef1b0dd7976c4e8eb13d8946ec699ae4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/42f2f81997d945bfabf96ff4cad94c7c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e9de676ee33a474a932919299e7233c1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6e91bd6512b45dc88267fefcce96a3d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bf146e555f114543b017cd9d8e04a6fb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bed41a0f6b8b4d2e8ab057d6f07af6ca, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f7967477466b497d98a04a828206dafd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/de6dd82725d246dfa97349b6ed2b4597, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/735d7041b64a472583540642e0e50eb0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/371c9153b08f4a759bc3314975766bbc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/33efec35d24f4423a5760d3b026edfd9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e426233b5e2a47139a506c004d0819e9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/62fc93e1d07d4c90924ff463ae879609, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6847df1703e49158cbfd97bcfb2d5d7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4e16b95adf814ff4be8a2e8675bd6852, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c5398fd5d6f04dd8abb86dc6ce1e52e8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/55c0e289c196456587738d0efbecb3b8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/5ea9edc9cb2b4681a54fbae7775f361e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/71b7880b90e24e008704f2057cd50266, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4da472787ada4d629d23d58fc0d7416f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d331b31bed1c4fb6b8e83c93cdbaa047, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/60c677126a59488196102e7e87cb28d6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef04547df30e4b43be061b11139cedda, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4ed677b749e14459ab4ddf192dfcd254, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/cdaad31fe4574a9fa093c032809640ad] to archive 2024-11-25T03:14:18,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:14:18,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1657a4c67ddc4c51836161bc0d80804c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1657a4c67ddc4c51836161bc0d80804c 2024-11-25T03:14:18,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/211b1461f4de41e9b6e569d1d745b508 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/211b1461f4de41e9b6e569d1d745b508 2024-11-25T03:14:18,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f3ae286bdabf4954a1c1518481ec511a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f3ae286bdabf4954a1c1518481ec511a 2024-11-25T03:14:18,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/a9a52a3cde8442a0a0ce2066718c9e96 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/a9a52a3cde8442a0a0ce2066718c9e96 2024-11-25T03:14:18,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4611b49c14cc4873ab00342b7b61cc03 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4611b49c14cc4873ab00342b7b61cc03 2024-11-25T03:14:18,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/47f83ba0578b4f1c83093dd90fb1ca68 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/47f83ba0578b4f1c83093dd90fb1ca68 2024-11-25T03:14:18,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/dd2123c81f4b4d77ae908317874c237a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/dd2123c81f4b4d77ae908317874c237a 2024-11-25T03:14:18,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/fd160a6838aa4e64a6881b80a72a47ee to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/fd160a6838aa4e64a6881b80a72a47ee 2024-11-25T03:14:18,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/8462d576bb324b4cb34ad324b77d829a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/8462d576bb324b4cb34ad324b77d829a 2024-11-25T03:14:18,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d850b17fbbd547aaad73e06e87804169 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d850b17fbbd547aaad73e06e87804169 2024-11-25T03:14:18,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef1b0dd7976c4e8eb13d8946ec699ae4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef1b0dd7976c4e8eb13d8946ec699ae4 2024-11-25T03:14:18,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/42f2f81997d945bfabf96ff4cad94c7c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/42f2f81997d945bfabf96ff4cad94c7c 2024-11-25T03:14:18,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e9de676ee33a474a932919299e7233c1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e9de676ee33a474a932919299e7233c1 2024-11-25T03:14:18,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6e91bd6512b45dc88267fefcce96a3d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6e91bd6512b45dc88267fefcce96a3d 2024-11-25T03:14:18,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bf146e555f114543b017cd9d8e04a6fb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bf146e555f114543b017cd9d8e04a6fb 2024-11-25T03:14:18,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bed41a0f6b8b4d2e8ab057d6f07af6ca to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/bed41a0f6b8b4d2e8ab057d6f07af6ca 2024-11-25T03:14:18,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f7967477466b497d98a04a828206dafd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/f7967477466b497d98a04a828206dafd 2024-11-25T03:14:18,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/de6dd82725d246dfa97349b6ed2b4597 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/de6dd82725d246dfa97349b6ed2b4597 2024-11-25T03:14:18,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/735d7041b64a472583540642e0e50eb0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/735d7041b64a472583540642e0e50eb0 2024-11-25T03:14:18,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/371c9153b08f4a759bc3314975766bbc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/371c9153b08f4a759bc3314975766bbc 2024-11-25T03:14:18,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/33efec35d24f4423a5760d3b026edfd9 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/33efec35d24f4423a5760d3b026edfd9 2024-11-25T03:14:18,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e426233b5e2a47139a506c004d0819e9 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/e426233b5e2a47139a506c004d0819e9 2024-11-25T03:14:18,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/62fc93e1d07d4c90924ff463ae879609 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/62fc93e1d07d4c90924ff463ae879609 2024-11-25T03:14:18,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6847df1703e49158cbfd97bcfb2d5d7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c6847df1703e49158cbfd97bcfb2d5d7 2024-11-25T03:14:18,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4e16b95adf814ff4be8a2e8675bd6852 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4e16b95adf814ff4be8a2e8675bd6852 2024-11-25T03:14:18,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c5398fd5d6f04dd8abb86dc6ce1e52e8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/c5398fd5d6f04dd8abb86dc6ce1e52e8 2024-11-25T03:14:18,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/55c0e289c196456587738d0efbecb3b8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/55c0e289c196456587738d0efbecb3b8 2024-11-25T03:14:18,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/5ea9edc9cb2b4681a54fbae7775f361e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/5ea9edc9cb2b4681a54fbae7775f361e 2024-11-25T03:14:18,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/71b7880b90e24e008704f2057cd50266 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/71b7880b90e24e008704f2057cd50266 2024-11-25T03:14:18,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4da472787ada4d629d23d58fc0d7416f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4da472787ada4d629d23d58fc0d7416f 2024-11-25T03:14:18,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d331b31bed1c4fb6b8e83c93cdbaa047 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/d331b31bed1c4fb6b8e83c93cdbaa047 2024-11-25T03:14:18,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/60c677126a59488196102e7e87cb28d6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/60c677126a59488196102e7e87cb28d6 2024-11-25T03:14:18,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef04547df30e4b43be061b11139cedda to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ef04547df30e4b43be061b11139cedda 2024-11-25T03:14:18,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4ed677b749e14459ab4ddf192dfcd254 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/4ed677b749e14459ab4ddf192dfcd254 2024-11-25T03:14:18,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/cdaad31fe4574a9fa093c032809640ad to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/cdaad31fe4574a9fa093c032809640ad 2024-11-25T03:14:18,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/4c312a25f6c744668e7a9cf22de433d2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/fadf92afd3f84902adc570df143943e4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3bdfd9911bcc425ab268f25d6262c6ea, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/0c9f9bf207a442e3be4896b94fe7c2bd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/dc3ca30e58b04b14ae62c6e7988708af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffec0c209bfa4e16a6dae6e569b8ffc6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/70e824c2c6084d1d85b290b8c54a70eb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a9f1e59232e045af98af2941fa71770a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/497525ee439b47fd8398e8d3d6a325e8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/df58ee78a3434df7890fbe4eaa731aba, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/407afd90567e49528437378d959aeeaa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/457a2ab9b81d41ae9f9adc080d69f433, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7a253de9d790446585bf960b47e2e961, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/107af241e0064457959f9e0198ead7a7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/40f3d48f430d454ca1a6c1138f17c94b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/44987291156c4a92a52a0b760cbca394, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/31bd35bec7e145b1a6eefb2f1a981981, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9c320180704747d6a1f7688fbec522d4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffbebe6ea29341f3b8e9db4b502bb40e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/83738761c5a74d2281cdc60be190cbaa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d818a6a14ec645069d6dcd5bd78a5a1c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f39c0a92f1164c368b5fd2787da5e508, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/5b78720d6df2478cb1f8a37c97361cbd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/96ea9d402c794dcd98e027bc34bf36e8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/91a981ebde504427911fd2ec5bcf7e1f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ed1c7bf82b674a5d804314f38bd347f4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/b6aa34433202446dbcadd9873e343198, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/43ca07b787474a64a3e50955d842f9cd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/90a036bfe6764cec9b3aa26f36386230, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a0392edbac7b40f0be2e2c3a265b5cdb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f69c2a970112464a8cbda95b0a1b5f20, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7ae025de686746b3b72dffa81e7246fd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/c9f09e545a2c4a5587519ad2b8bc8e68, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ea1aea42e0054a4bb1be5cf36abadaef] to archive 2024-11-25T03:14:18,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:14:18,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/4c312a25f6c744668e7a9cf22de433d2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/4c312a25f6c744668e7a9cf22de433d2 2024-11-25T03:14:18,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/fadf92afd3f84902adc570df143943e4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/fadf92afd3f84902adc570df143943e4 2024-11-25T03:14:18,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3bdfd9911bcc425ab268f25d6262c6ea to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3bdfd9911bcc425ab268f25d6262c6ea 2024-11-25T03:14:18,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/0c9f9bf207a442e3be4896b94fe7c2bd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/0c9f9bf207a442e3be4896b94fe7c2bd 2024-11-25T03:14:18,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/dc3ca30e58b04b14ae62c6e7988708af to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/dc3ca30e58b04b14ae62c6e7988708af 2024-11-25T03:14:18,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffec0c209bfa4e16a6dae6e569b8ffc6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffec0c209bfa4e16a6dae6e569b8ffc6 2024-11-25T03:14:18,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/70e824c2c6084d1d85b290b8c54a70eb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/70e824c2c6084d1d85b290b8c54a70eb 2024-11-25T03:14:18,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a9f1e59232e045af98af2941fa71770a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a9f1e59232e045af98af2941fa71770a 2024-11-25T03:14:18,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/497525ee439b47fd8398e8d3d6a325e8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/497525ee439b47fd8398e8d3d6a325e8 2024-11-25T03:14:18,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/df58ee78a3434df7890fbe4eaa731aba to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/df58ee78a3434df7890fbe4eaa731aba 2024-11-25T03:14:18,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/407afd90567e49528437378d959aeeaa to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/407afd90567e49528437378d959aeeaa 2024-11-25T03:14:18,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/457a2ab9b81d41ae9f9adc080d69f433 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/457a2ab9b81d41ae9f9adc080d69f433 2024-11-25T03:14:18,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7a253de9d790446585bf960b47e2e961 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7a253de9d790446585bf960b47e2e961 2024-11-25T03:14:18,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/107af241e0064457959f9e0198ead7a7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/107af241e0064457959f9e0198ead7a7 2024-11-25T03:14:18,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3c0eaf00c7b747e9a46a1f8fb6e9e4a8 2024-11-25T03:14:18,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/40f3d48f430d454ca1a6c1138f17c94b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/40f3d48f430d454ca1a6c1138f17c94b 2024-11-25T03:14:18,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/44987291156c4a92a52a0b760cbca394 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/44987291156c4a92a52a0b760cbca394 2024-11-25T03:14:18,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/31bd35bec7e145b1a6eefb2f1a981981 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/31bd35bec7e145b1a6eefb2f1a981981 2024-11-25T03:14:18,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9c320180704747d6a1f7688fbec522d4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9c320180704747d6a1f7688fbec522d4 2024-11-25T03:14:18,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffbebe6ea29341f3b8e9db4b502bb40e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ffbebe6ea29341f3b8e9db4b502bb40e 2024-11-25T03:14:18,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/83738761c5a74d2281cdc60be190cbaa to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/83738761c5a74d2281cdc60be190cbaa 2024-11-25T03:14:18,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d818a6a14ec645069d6dcd5bd78a5a1c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d818a6a14ec645069d6dcd5bd78a5a1c 2024-11-25T03:14:18,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f39c0a92f1164c368b5fd2787da5e508 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f39c0a92f1164c368b5fd2787da5e508 2024-11-25T03:14:18,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/5b78720d6df2478cb1f8a37c97361cbd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/5b78720d6df2478cb1f8a37c97361cbd 2024-11-25T03:14:18,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/96ea9d402c794dcd98e027bc34bf36e8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/96ea9d402c794dcd98e027bc34bf36e8 2024-11-25T03:14:18,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/91a981ebde504427911fd2ec5bcf7e1f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/91a981ebde504427911fd2ec5bcf7e1f 2024-11-25T03:14:18,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ed1c7bf82b674a5d804314f38bd347f4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ed1c7bf82b674a5d804314f38bd347f4 2024-11-25T03:14:18,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/b6aa34433202446dbcadd9873e343198 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/b6aa34433202446dbcadd9873e343198 2024-11-25T03:14:18,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/43ca07b787474a64a3e50955d842f9cd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/43ca07b787474a64a3e50955d842f9cd 2024-11-25T03:14:18,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/90a036bfe6764cec9b3aa26f36386230 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/90a036bfe6764cec9b3aa26f36386230 2024-11-25T03:14:18,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a0392edbac7b40f0be2e2c3a265b5cdb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/a0392edbac7b40f0be2e2c3a265b5cdb 2024-11-25T03:14:18,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f69c2a970112464a8cbda95b0a1b5f20 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/f69c2a970112464a8cbda95b0a1b5f20 2024-11-25T03:14:18,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7ae025de686746b3b72dffa81e7246fd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/7ae025de686746b3b72dffa81e7246fd 2024-11-25T03:14:18,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/c9f09e545a2c4a5587519ad2b8bc8e68 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/c9f09e545a2c4a5587519ad2b8bc8e68 2024-11-25T03:14:18,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ea1aea42e0054a4bb1be5cf36abadaef to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/ea1aea42e0054a4bb1be5cf36abadaef 2024-11-25T03:14:18,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8877e1f6602244768ef8f512e4b822b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/403a2b81f2b44f1f8c132d8aa43cc54c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/51891bf05ea943a481faa85b276ef7cc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b1bcee6a57b46aca04cec80004e142b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/999096c7e6e0414189067ed877340314, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1bd1db95ea05430ea73219331e3e9bb8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/079ff8cc819c41c98c653fe891c33a5d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/2ea5b64c5c9f4137b0f9a9835f55ecdb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8f602b6706f4e7c8075f3412d88d2c5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/a2114f2058064228abbd0cf2d9bfea49, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/31d36055103b492bb7ef4289f37b656a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/5369c9fb9e164a1d86324fc83e90e001, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/4288ac7f75274a6fbfd11935fe48cb37, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/3126f9b5ef8e45f581eb0f9ad42d00e5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c600bbbc0f4a426e9219510e5eb735d2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c0ae31c94e4542b392c4293e2e6fa5a0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/fcf40bb0e1454fc78438c0acbe51a4bd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8391bcebce74f21b6c8b08fb11afbd3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/9c7f0d2c590b4827b4df71e22b0dbd31, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e5f87a13d6254f04a9213699dd88177c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8830750fa2fe4a14a01442c2a8d5acb3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/cb4629418db4482bad6bb20fd05042fa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b648de9a2944d61a9758397f9dbe1b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8a3a23131a5e40ea848143e677b05a60, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/03c576035b964880a142b71f8ce321a3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/452e2f3784ff4b969089f145f5dbebb0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e437aa3b47594eb59c547b229fe5c873, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/38818c3fb71640e2bbd89983d95b5bb0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/21a8c4ecb3e94e3fa8ef4b68750aedb9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/22ff8e1cbb7c4a90a52af988a5f67d94, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/bb5d72510444456d8c75650098726b6d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/757d1a99979e4255a8f33f62825fe1cb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a085f2416e4330adafb239f7ff45fe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a55848542a483baf0422298da1d918, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8cf9a83b790b4644b571715bc1390dad] to archive 2024-11-25T03:14:18,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:14:18,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8877e1f6602244768ef8f512e4b822b3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8877e1f6602244768ef8f512e4b822b3 2024-11-25T03:14:18,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/403a2b81f2b44f1f8c132d8aa43cc54c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/403a2b81f2b44f1f8c132d8aa43cc54c 2024-11-25T03:14:18,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/51891bf05ea943a481faa85b276ef7cc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/51891bf05ea943a481faa85b276ef7cc 2024-11-25T03:14:18,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b1bcee6a57b46aca04cec80004e142b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b1bcee6a57b46aca04cec80004e142b 2024-11-25T03:14:18,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/999096c7e6e0414189067ed877340314 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/999096c7e6e0414189067ed877340314 2024-11-25T03:14:18,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1bd1db95ea05430ea73219331e3e9bb8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1bd1db95ea05430ea73219331e3e9bb8 2024-11-25T03:14:18,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/079ff8cc819c41c98c653fe891c33a5d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/079ff8cc819c41c98c653fe891c33a5d 2024-11-25T03:14:18,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/2ea5b64c5c9f4137b0f9a9835f55ecdb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/2ea5b64c5c9f4137b0f9a9835f55ecdb 2024-11-25T03:14:18,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8f602b6706f4e7c8075f3412d88d2c5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8f602b6706f4e7c8075f3412d88d2c5 2024-11-25T03:14:18,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/a2114f2058064228abbd0cf2d9bfea49 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/a2114f2058064228abbd0cf2d9bfea49 2024-11-25T03:14:18,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/31d36055103b492bb7ef4289f37b656a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/31d36055103b492bb7ef4289f37b656a 2024-11-25T03:14:18,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/5369c9fb9e164a1d86324fc83e90e001 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/5369c9fb9e164a1d86324fc83e90e001 2024-11-25T03:14:18,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/4288ac7f75274a6fbfd11935fe48cb37 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/4288ac7f75274a6fbfd11935fe48cb37 2024-11-25T03:14:18,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/3126f9b5ef8e45f581eb0f9ad42d00e5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/3126f9b5ef8e45f581eb0f9ad42d00e5 2024-11-25T03:14:18,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c600bbbc0f4a426e9219510e5eb735d2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c600bbbc0f4a426e9219510e5eb735d2 2024-11-25T03:14:18,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c0ae31c94e4542b392c4293e2e6fa5a0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c0ae31c94e4542b392c4293e2e6fa5a0 2024-11-25T03:14:18,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/fcf40bb0e1454fc78438c0acbe51a4bd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/fcf40bb0e1454fc78438c0acbe51a4bd 2024-11-25T03:14:18,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8391bcebce74f21b6c8b08fb11afbd3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/c8391bcebce74f21b6c8b08fb11afbd3 2024-11-25T03:14:18,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/9c7f0d2c590b4827b4df71e22b0dbd31 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/9c7f0d2c590b4827b4df71e22b0dbd31 2024-11-25T03:14:18,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e5f87a13d6254f04a9213699dd88177c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e5f87a13d6254f04a9213699dd88177c 2024-11-25T03:14:18,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8830750fa2fe4a14a01442c2a8d5acb3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8830750fa2fe4a14a01442c2a8d5acb3 2024-11-25T03:14:18,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/cb4629418db4482bad6bb20fd05042fa to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/cb4629418db4482bad6bb20fd05042fa 2024-11-25T03:14:18,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b648de9a2944d61a9758397f9dbe1b3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/1b648de9a2944d61a9758397f9dbe1b3 2024-11-25T03:14:18,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8a3a23131a5e40ea848143e677b05a60 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8a3a23131a5e40ea848143e677b05a60 2024-11-25T03:14:18,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/03c576035b964880a142b71f8ce321a3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/03c576035b964880a142b71f8ce321a3 2024-11-25T03:14:18,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/452e2f3784ff4b969089f145f5dbebb0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/452e2f3784ff4b969089f145f5dbebb0 2024-11-25T03:14:18,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e437aa3b47594eb59c547b229fe5c873 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/e437aa3b47594eb59c547b229fe5c873 2024-11-25T03:14:18,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/38818c3fb71640e2bbd89983d95b5bb0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/38818c3fb71640e2bbd89983d95b5bb0 2024-11-25T03:14:18,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/21a8c4ecb3e94e3fa8ef4b68750aedb9 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/21a8c4ecb3e94e3fa8ef4b68750aedb9 2024-11-25T03:14:18,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/22ff8e1cbb7c4a90a52af988a5f67d94 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/22ff8e1cbb7c4a90a52af988a5f67d94 2024-11-25T03:14:18,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/bb5d72510444456d8c75650098726b6d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/bb5d72510444456d8c75650098726b6d 2024-11-25T03:14:18,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/757d1a99979e4255a8f33f62825fe1cb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/757d1a99979e4255a8f33f62825fe1cb 2024-11-25T03:14:18,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a085f2416e4330adafb239f7ff45fe to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a085f2416e4330adafb239f7ff45fe 2024-11-25T03:14:18,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a55848542a483baf0422298da1d918 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/32a55848542a483baf0422298da1d918 2024-11-25T03:14:18,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8cf9a83b790b4644b571715bc1390dad to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/8cf9a83b790b4644b571715bc1390dad 2024-11-25T03:14:18,953 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/recovered.edits/507.seqid, newMaxSeqId=507, maxSeqId=1 2024-11-25T03:14:18,954 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. 2024-11-25T03:14:18,954 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 159c9b03400b7fa4e53f5245dc0f2c9f: Waiting for close lock at 1732504456524Running coprocessor pre-close hooks at 1732504456524Disabling compacts and flushes for region at 1732504456524Disabling writes for close at 1732504457549 (+1025 ms)Obtaining lock to block concurrent updates at 1732504457549Preparing flush snapshotting stores in 159c9b03400b7fa4e53f5245dc0f2c9f at 1732504457549Finished memstore snapshotting TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732504457549Flushing stores of TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f. at 1732504457550 (+1 ms)Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/A: creating writer at 1732504457550Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/A: appending metadata at 1732504457552 (+2 ms)Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/A: closing flushed file at 1732504457553 (+1 ms)Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/B: creating writer at 1732504457963 (+410 ms)Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/B: appending metadata at 1732504457965 (+2 ms)Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/B: closing flushed file at 1732504457965Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/C: creating writer at 1732504458375 (+410 ms)Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/C: appending metadata at 1732504458377 (+2 ms)Flushing 159c9b03400b7fa4e53f5245dc0f2c9f/C: closing flushed file at 1732504458377Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d88460: reopening flushed file at 1732504458787 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47b2d234: reopening flushed file at 1732504458791 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@161e770c: reopening flushed file at 1732504458795 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 159c9b03400b7fa4e53f5245dc0f2c9f in 1251ms, sequenceid=504, compaction requested=true at 1732504458800 (+5 ms)Writing region close event to WAL at 1732504458949 (+149 ms)Running coprocessor post-close hooks at 1732504458953 (+4 ms)Closed at 1732504458953 2024-11-25T03:14:18,957 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:18,958 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=159c9b03400b7fa4e53f5245dc0f2c9f, regionState=CLOSED 2024-11-25T03:14:18,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:14:18,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-25T03:14:18,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 159c9b03400b7fa4e53f5245dc0f2c9f, server=38a5d2077268,43357,1732504425108 in 2.5940 sec 2024-11-25T03:14:18,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-25T03:14:18,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=159c9b03400b7fa4e53f5245dc0f2c9f, UNASSIGN in 2.6000 sec 2024-11-25T03:14:18,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T03:14:18,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6070 sec 2024-11-25T03:14:18,968 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504458968"}]},"ts":"1732504458968"} 2024-11-25T03:14:18,970 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:14:18,970 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:14:18,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6220 sec 2024-11-25T03:14:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:14:20,530 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:20,533 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:14:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:20,539 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-25T03:14:20,541 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:20,547 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:20,552 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/recovered.edits] 2024-11-25T03:14:20,555 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/01311ef1ac264a3183de6b109bb4ebae to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/01311ef1ac264a3183de6b109bb4ebae 2024-11-25T03:14:20,557 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/0aa3afb9821d4429a22e0114e68ab19e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/0aa3afb9821d4429a22e0114e68ab19e 2024-11-25T03:14:20,558 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1aafd632e0b046d688b81fb23f97cdd3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/1aafd632e0b046d688b81fb23f97cdd3 2024-11-25T03:14:20,559 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ac096d5ac2ee49c2a91475c03ef2c7f6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ac096d5ac2ee49c2a91475c03ef2c7f6 2024-11-25T03:14:20,561 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ddcc32f0872e4cb7a0fcdc0e83f09eb9 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/A/ddcc32f0872e4cb7a0fcdc0e83f09eb9 2024-11-25T03:14:20,563 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/22e864c1c70542a096bbd90afd0c7a09 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/22e864c1c70542a096bbd90afd0c7a09 2024-11-25T03:14:20,565 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3232bfbfc94347fcafd0cda51db382bc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3232bfbfc94347fcafd0cda51db382bc 2024-11-25T03:14:20,566 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3e34752d37a948c799637aed43e1d569 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/3e34752d37a948c799637aed43e1d569 2024-11-25T03:14:20,567 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9aaf05bcdefd4678ae852d3b1fd383b3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/9aaf05bcdefd4678ae852d3b1fd383b3 2024-11-25T03:14:20,569 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d718fdb839a447f19943241306fc170f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/B/d718fdb839a447f19943241306fc170f 2024-11-25T03:14:20,571 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/79a79cd2f16142daaea2f55c5dd211e7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/79a79cd2f16142daaea2f55c5dd211e7 2024-11-25T03:14:20,573 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/b6efb30cf6ff4fa4bc6db7df300b26f8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/b6efb30cf6ff4fa4bc6db7df300b26f8 2024-11-25T03:14:20,574 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/b9f2001e0c994efb9a1d710610bf2a71 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/b9f2001e0c994efb9a1d710610bf2a71 2024-11-25T03:14:20,576 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f40aecfbccb148d587b9248417ef14a5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f40aecfbccb148d587b9248417ef14a5 2024-11-25T03:14:20,577 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f46cd3ad77234a7c900adb127bf840aa to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/C/f46cd3ad77234a7c900adb127bf840aa 2024-11-25T03:14:20,581 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/recovered.edits/507.seqid to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f/recovered.edits/507.seqid 2024-11-25T03:14:20,581 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/159c9b03400b7fa4e53f5245dc0f2c9f 2024-11-25T03:14:20,582 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:14:20,587 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:20,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-25T03:14:20,598 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:14:20,638 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:14:20,640 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:20,640 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:14:20,640 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732504460640"}]},"ts":"9223372036854775807"} 2024-11-25T03:14:20,646 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:14:20,646 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 159c9b03400b7fa4e53f5245dc0f2c9f, NAME => 'TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:14:20,646 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:14:20,646 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732504460646"}]},"ts":"9223372036854775807"} 2024-11-25T03:14:20,649 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:14:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-25T03:14:20,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:20,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 116 msec 2024-11-25T03:14:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-25T03:14:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:14:20,861 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:20,874 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=243 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;38a5d2077268:43357-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-34767401_22 at /127.0.0.1:52504 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 277) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7586 (was 7375) - AvailableMemoryMB LEAK? - 2024-11-25T03:14:20,883 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=243, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=7586 2024-11-25T03:14:20,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:14:20,885 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:14:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:20,887 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:14:20,887 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:20,887 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-25T03:14:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:14:20,888 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:14:20,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741957_1133 (size=1061) 2024-11-25T03:14:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:14:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:14:21,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:14:21,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741958_1134 (size=53) 2024-11-25T03:14:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:14:21,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:14:21,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 73fa306731753442713ff2a6a1c4403b, disabling compactions & flushes 2024-11-25T03:14:21,702 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:21,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:21,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. after waiting 0 ms 2024-11-25T03:14:21,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:21,702 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:21,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 73fa306731753442713ff2a6a1c4403b: Waiting for close lock at 1732504461702Disabling compacts and flushes for region at 1732504461702Disabling writes for close at 1732504461702Writing region close event to WAL at 1732504461702Closed at 1732504461702 2024-11-25T03:14:21,704 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:14:21,704 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732504461704"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732504461704"}]},"ts":"1732504461704"} 2024-11-25T03:14:21,706 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:14:21,707 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:14:21,707 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504461707"}]},"ts":"1732504461707"} 2024-11-25T03:14:21,708 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:14:21,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, ASSIGN}] 2024-11-25T03:14:21,710 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, ASSIGN 2024-11-25T03:14:21,711 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, ASSIGN; state=OFFLINE, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=false 2024-11-25T03:14:21,861 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:21,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, ASSIGN because future has completed 2024-11-25T03:14:21,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:14:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:14:22,020 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:22,021 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:14:22,021 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,021 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:14:22,021 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,021 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,023 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,024 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:22,025 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73fa306731753442713ff2a6a1c4403b columnFamilyName A 2024-11-25T03:14:22,025 DEBUG [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:22,026 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(327): Store=73fa306731753442713ff2a6a1c4403b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:22,026 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,027 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:22,027 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73fa306731753442713ff2a6a1c4403b columnFamilyName B 2024-11-25T03:14:22,027 DEBUG [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:22,027 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(327): Store=73fa306731753442713ff2a6a1c4403b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:22,027 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,028 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:22,029 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73fa306731753442713ff2a6a1c4403b columnFamilyName C 2024-11-25T03:14:22,029 DEBUG [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:22,029 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(327): Store=73fa306731753442713ff2a6a1c4403b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:22,029 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:22,029 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,030 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,030 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,031 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,031 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,032 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:14:22,033 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,035 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:14:22,035 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 73fa306731753442713ff2a6a1c4403b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62353026, jitterRate=-0.07086750864982605}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:14:22,035 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:22,036 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 73fa306731753442713ff2a6a1c4403b: Running coprocessor pre-open hook at 1732504462021Writing region info on filesystem at 1732504462021Initializing all the Stores at 1732504462022 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504462022Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504462022Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504462023 (+1 ms)Cleaning up temporary data from old regions at 1732504462031 (+8 ms)Running coprocessor post-open hooks at 1732504462035 (+4 ms)Region opened successfully at 1732504462036 (+1 ms) 2024-11-25T03:14:22,037 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., pid=38, masterSystemTime=1732504462016 2024-11-25T03:14:22,039 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:22,039 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:22,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:22,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:14:22,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-25T03:14:22,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 in 178 msec 2024-11-25T03:14:22,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-25T03:14:22,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, ASSIGN in 335 msec 2024-11-25T03:14:22,047 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:14:22,047 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504462047"}]},"ts":"1732504462047"} 2024-11-25T03:14:22,049 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:14:22,049 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:14:22,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-25T03:14:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:14:23,030 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:23,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:23,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:23,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:23,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5878bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:23,036 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478c3bf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:23,037 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:23,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:23,041 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:14:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,44623,1732504424313 2024-11-25T03:14:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4dcf76b9 2024-11-25T03:14:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:14:23,043 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:14:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:14:23,048 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:14:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:23,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741959_1135 (size=1097) 2024-11-25T03:14:23,466 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T03:14:23,467 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T03:14:23,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:14:23,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, REOPEN/MOVE}] 2024-11-25T03:14:23,478 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, REOPEN/MOVE 2024-11-25T03:14:23,479 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:23,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, REOPEN/MOVE because future has completed 2024-11-25T03:14:23,481 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:14:23,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:14:23,633 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,633 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:14:23,634 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 73fa306731753442713ff2a6a1c4403b, disabling compactions & flushes 2024-11-25T03:14:23,634 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,634 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,634 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. after waiting 0 ms 2024-11-25T03:14:23,634 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,638 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:14:23,639 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,639 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 73fa306731753442713ff2a6a1c4403b: Waiting for close lock at 1732504463633Running coprocessor pre-close hooks at 1732504463633Disabling compacts and flushes for region at 1732504463633Disabling writes for close at 1732504463634 (+1 ms)Writing region close event to WAL at 1732504463634Running coprocessor post-close hooks at 1732504463639 (+5 ms)Closed at 1732504463639 2024-11-25T03:14:23,639 WARN [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 73fa306731753442713ff2a6a1c4403b to self. 2024-11-25T03:14:23,641 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,642 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=CLOSED 2024-11-25T03:14:23,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:14:23,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-25T03:14:23,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 in 163 msec 2024-11-25T03:14:23,646 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, REOPEN/MOVE; state=CLOSED, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=true 2024-11-25T03:14:23,797 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:23,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, REOPEN/MOVE because future has completed 2024-11-25T03:14:23,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:14:23,955 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,955 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:14:23,955 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,955 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:14:23,955 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,955 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,960 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,961 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:23,966 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73fa306731753442713ff2a6a1c4403b columnFamilyName A 2024-11-25T03:14:23,968 DEBUG [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:23,969 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(327): Store=73fa306731753442713ff2a6a1c4403b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:23,969 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,970 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:23,970 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73fa306731753442713ff2a6a1c4403b columnFamilyName B 2024-11-25T03:14:23,970 DEBUG [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:23,971 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(327): Store=73fa306731753442713ff2a6a1c4403b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:23,971 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,972 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:23,972 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73fa306731753442713ff2a6a1c4403b columnFamilyName C 2024-11-25T03:14:23,972 DEBUG [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:23,973 INFO [StoreOpener-73fa306731753442713ff2a6a1c4403b-1 {}] regionserver.HStore(327): Store=73fa306731753442713ff2a6a1c4403b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:23,973 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,973 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,974 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,975 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,976 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,976 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,977 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:14:23,978 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,979 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 73fa306731753442713ff2a6a1c4403b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74445450, jitterRate=0.109323650598526}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:14:23,979 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:23,980 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 73fa306731753442713ff2a6a1c4403b: Running coprocessor pre-open hook at 1732504463956Writing region info on filesystem at 1732504463956Initializing all the Stores at 1732504463957 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504463957Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504463957Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504463957Cleaning up temporary data from old regions at 1732504463976 (+19 ms)Running coprocessor post-open hooks at 1732504463979 (+3 ms)Region opened successfully at 1732504463980 (+1 ms) 2024-11-25T03:14:23,981 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., pid=43, masterSystemTime=1732504463951 2024-11-25T03:14:23,983 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,983 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:23,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=OPEN, openSeqNum=5, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:23,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:14:23,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-25T03:14:23,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 in 187 msec 2024-11-25T03:14:23,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-25T03:14:23,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, REOPEN/MOVE in 511 msec 2024-11-25T03:14:23,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-25T03:14:23,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-25T03:14:23,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-11-25T03:14:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T03:14:23,998 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:23,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:23,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:23,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7357a70d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,002 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a209e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,005 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9c90f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,006 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b23d02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,009 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67812d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,011 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50858797, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b15085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,015 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cea8b3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,017 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee8b6d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,019 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491f1db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,021 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc7fc3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752fcdd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5115bbaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,027 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1582771d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,029 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad0a601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,030 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787a4f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:24,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:24,033 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50fb8148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:24,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e410ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:24,036 DEBUG [Thread-683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,037 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:24,037 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,037 DEBUG [Thread-689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,038 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-25T03:14:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:14:24,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:24,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:24,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:24,041 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,042 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,045 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,049 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,053 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:24,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:14:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:24,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:24,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dcd0fd9e6bec4973881288c764bcb14b_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504464068/Put/seqid=0 2024-11-25T03:14:24,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741960_1136 (size=12154) 2024-11-25T03:14:24,147 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:14:24,153 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dcd0fd9e6bec4973881288c764bcb14b_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dcd0fd9e6bec4973881288c764bcb14b_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:24,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/e302c7e28462441fb4dd859f1e4b7bc1, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:24,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/e302c7e28462441fb4dd859f1e4b7bc1 is 175, key is test_row_0/A:col10/1732504464068/Put/seqid=0 2024-11-25T03:14:24,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732504464111, max timestamp=1732504464157Num uniques 859; 2024-11-25T03:14:24,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732504464121, max timestamp=1732504464157Num uniques 859; 2024-11-25T03:14:24,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-25T03:14:24,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-25T03:14:24,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741961_1137 (size=30955) 2024-11-25T03:14:24,193 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/e302c7e28462441fb4dd859f1e4b7bc1 2024-11-25T03:14:24,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732504464121, max timestamp=1732504464157Num uniques 859; 2024-11-25T03:14:24,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T03:14:24,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:14:24,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:24,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504524208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504524210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504524213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504524216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/d85b101c8296496d911a1ae676a8dd09 is 50, key is test_row_0/B:col10/1732504464068/Put/seqid=0 2024-11-25T03:14:24,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504524225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741962_1138 (size=12001) 2024-11-25T03:14:24,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/d85b101c8296496d911a1ae676a8dd09 2024-11-25T03:14:24,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/33ab486886df4e209f8148bc40c96aac is 50, key is test_row_0/C:col10/1732504464068/Put/seqid=0 2024-11-25T03:14:24,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741963_1139 (size=12001) 2024-11-25T03:14:24,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504524330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/33ab486886df4e209f8148bc40c96aac 2024-11-25T03:14:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504524331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/e302c7e28462441fb4dd859f1e4b7bc1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/e302c7e28462441fb4dd859f1e4b7bc1 2024-11-25T03:14:24,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504524331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504524331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504524341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:14:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/e302c7e28462441fb4dd859f1e4b7bc1, entries=150, sequenceid=15, filesize=30.2 K 2024-11-25T03:14:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:24,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/d85b101c8296496d911a1ae676a8dd09 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/d85b101c8296496d911a1ae676a8dd09 2024-11-25T03:14:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:24,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:24,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/d85b101c8296496d911a1ae676a8dd09, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:14:24,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/33ab486886df4e209f8148bc40c96aac as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/33ab486886df4e209f8148bc40c96aac 2024-11-25T03:14:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:14:24,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/33ab486886df4e209f8148bc40c96aac, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:14:24,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for 73fa306731753442713ff2a6a1c4403b in 301ms, sequenceid=15, compaction requested=false 2024-11-25T03:14:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:24,480 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:14:24,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:24,505 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=129.31 KB heapSize=355.16 KB 2024-11-25T03:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:24,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:24,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:24,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112592628547a835493288778e833e15dc7f_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504464206/Put/seqid=0 2024-11-25T03:14:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:24,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741964_1140 (size=12154) 2024-11-25T03:14:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504524557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504524560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504524560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504524571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504524574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:14:24,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504524681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504524681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504524690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504524691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504524880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504524890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504524890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504524910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504524915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:24,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:24,959 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112592628547a835493288778e833e15dc7f_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112592628547a835493288778e833e15dc7f_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c80f38d4fb0a4e458ab65f063b5d15b3, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c80f38d4fb0a4e458ab65f063b5d15b3 is 175, key is test_row_0/A:col10/1732504464206/Put/seqid=0 2024-11-25T03:14:24,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741965_1141 (size=30955) 2024-11-25T03:14:25,001 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c80f38d4fb0a4e458ab65f063b5d15b3 2024-11-25T03:14:25,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/930df25eac904f50a469a2f3f5f91706 is 50, key is test_row_0/B:col10/1732504464206/Put/seqid=0 2024-11-25T03:14:25,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741966_1142 (size=12001) 2024-11-25T03:14:25,052 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/930df25eac904f50a469a2f3f5f91706 2024-11-25T03:14:25,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/8a646ef51a844b9f8b9af821c595cadd is 50, key is test_row_0/C:col10/1732504464206/Put/seqid=0 2024-11-25T03:14:25,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741967_1143 (size=12001) 2024-11-25T03:14:25,092 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.39 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/8a646ef51a844b9f8b9af821c595cadd 2024-11-25T03:14:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c80f38d4fb0a4e458ab65f063b5d15b3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c80f38d4fb0a4e458ab65f063b5d15b3 2024-11-25T03:14:25,107 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c80f38d4fb0a4e458ab65f063b5d15b3, entries=150, sequenceid=48, filesize=30.2 K 2024-11-25T03:14:25,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/930df25eac904f50a469a2f3f5f91706 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/930df25eac904f50a469a2f3f5f91706 2024-11-25T03:14:25,116 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/930df25eac904f50a469a2f3f5f91706, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T03:14:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/8a646ef51a844b9f8b9af821c595cadd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/8a646ef51a844b9f8b9af821c595cadd 2024-11-25T03:14:25,127 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/8a646ef51a844b9f8b9af821c595cadd, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T03:14:25,128 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~355.12 KB/363640, currentSize=53.67 KB/54960 for 73fa306731753442713ff2a6a1c4403b in 623ms, sequenceid=48, compaction requested=false 2024-11-25T03:14:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-25T03:14:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-25T03:14:25,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-25T03:14:25,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0900 sec 2024-11-25T03:14:25,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0980 sec 2024-11-25T03:14:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:14:25,180 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:25,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-25T03:14:25,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:14:25,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:25,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:25,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T03:14:25,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:25,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:25,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:25,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:25,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:25,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:25,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112522c18540d26f4893a36676966856af6e_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504464571/Put/seqid=0 2024-11-25T03:14:25,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504525264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504525266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504525273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504525276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:14:25,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741968_1144 (size=17034) 2024-11-25T03:14:25,303 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:25,309 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112522c18540d26f4893a36676966856af6e_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522c18540d26f4893a36676966856af6e_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:25,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/b9992bb570e94316b1a3d77ca262be3f, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:25,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/b9992bb570e94316b1a3d77ca262be3f is 175, key is test_row_0/A:col10/1732504464571/Put/seqid=0 2024-11-25T03:14:25,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741969_1145 (size=48139) 2024-11-25T03:14:25,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504525380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504525390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504525391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504525391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504525401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:14:25,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504525590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504525600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504525601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504525601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/b9992bb570e94316b1a3d77ca262be3f 2024-11-25T03:14:25,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/da003b456d6446ec999ff47f7aa94424 is 50, key is test_row_0/B:col10/1732504464571/Put/seqid=0 2024-11-25T03:14:25,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741970_1146 (size=12001) 2024-11-25T03:14:25,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:14:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504525900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504525920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504525920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504525921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:25,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:25,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/da003b456d6446ec999ff47f7aa94424 2024-11-25T03:14:26,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/4818c87fd0f94d30a1daf4f02ce84db3 is 50, key is test_row_0/C:col10/1732504464571/Put/seqid=0 2024-11-25T03:14:26,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:26,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741971_1147 (size=12001) 2024-11-25T03:14:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:14:26,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504526420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504526420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:26,427 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 73fa306731753442713ff2a6a1c4403b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:26,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504526431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:26,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:26,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504526432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:26,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504526432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:26,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/4818c87fd0f94d30a1daf4f02ce84db3 2024-11-25T03:14:26,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/b9992bb570e94316b1a3d77ca262be3f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9992bb570e94316b1a3d77ca262be3f 2024-11-25T03:14:26,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9992bb570e94316b1a3d77ca262be3f, entries=250, sequenceid=61, filesize=47.0 K 2024-11-25T03:14:26,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/da003b456d6446ec999ff47f7aa94424 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/da003b456d6446ec999ff47f7aa94424 2024-11-25T03:14:26,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/da003b456d6446ec999ff47f7aa94424, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T03:14:26,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/4818c87fd0f94d30a1daf4f02ce84db3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/4818c87fd0f94d30a1daf4f02ce84db3 2024-11-25T03:14:26,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/4818c87fd0f94d30a1daf4f02ce84db3, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T03:14:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:26,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 73fa306731753442713ff2a6a1c4403b in 1523ms, sequenceid=61, compaction requested=true 2024-11-25T03:14:26,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:26,738 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:26,740 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:26,740 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/A is initiating minor compaction (all files) 2024-11-25T03:14:26,740 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/A in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,740 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/e302c7e28462441fb4dd859f1e4b7bc1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c80f38d4fb0a4e458ab65f063b5d15b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9992bb570e94316b1a3d77ca262be3f] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=107.5 K 2024-11-25T03:14:26,740 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,741 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/e302c7e28462441fb4dd859f1e4b7bc1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c80f38d4fb0a4e458ab65f063b5d15b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9992bb570e94316b1a3d77ca262be3f] 2024-11-25T03:14:26,741 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting e302c7e28462441fb4dd859f1e4b7bc1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732504464068 2024-11-25T03:14:26,742 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80f38d4fb0a4e458ab65f063b5d15b3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732504464206 2024-11-25T03:14:26,742 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9992bb570e94316b1a3d77ca262be3f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732504464545 2024-11-25T03:14:26,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:26,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:26,745 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:26,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:26,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:26,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:26,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:26,748 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:26,748 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/B is initiating minor compaction (all files) 2024-11-25T03:14:26,748 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/B in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,749 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/d85b101c8296496d911a1ae676a8dd09, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/930df25eac904f50a469a2f3f5f91706, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/da003b456d6446ec999ff47f7aa94424] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=35.2 K 2024-11-25T03:14:26,749 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d85b101c8296496d911a1ae676a8dd09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732504464068 2024-11-25T03:14:26,753 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 930df25eac904f50a469a2f3f5f91706, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732504464206 2024-11-25T03:14:26,755 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting da003b456d6446ec999ff47f7aa94424, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732504464571 2024-11-25T03:14:26,763 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:26,777 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125442c8c16d5d248e9832a0f2265fc688e_73fa306731753442713ff2a6a1c4403b store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:26,787 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#B#compaction#130 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:26,788 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125442c8c16d5d248e9832a0f2265fc688e_73fa306731753442713ff2a6a1c4403b, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:26,788 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/ef25de7fcee840bdb1fcc69da6688d86 is 50, key is test_row_0/B:col10/1732504464571/Put/seqid=0 2024-11-25T03:14:26,788 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125442c8c16d5d248e9832a0f2265fc688e_73fa306731753442713ff2a6a1c4403b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:26,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741972_1148 (size=12104) 2024-11-25T03:14:26,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741973_1149 (size=4469) 2024-11-25T03:14:26,860 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#A#compaction#129 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:26,861 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/b9a9b6d6b2564ef288731a36d81a2172 is 175, key is test_row_0/A:col10/1732504464571/Put/seqid=0 2024-11-25T03:14:26,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:14:26,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:26,889 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:14:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:26,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:26,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741974_1150 (size=31058) 2024-11-25T03:14:26,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112543c6ba20653a40e184b12870b28d4a21_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504465271/Put/seqid=0 2024-11-25T03:14:26,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741975_1151 (size=12154) 2024-11-25T03:14:27,250 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/ef25de7fcee840bdb1fcc69da6688d86 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef25de7fcee840bdb1fcc69da6688d86 2024-11-25T03:14:27,265 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/B of 73fa306731753442713ff2a6a1c4403b into ef25de7fcee840bdb1fcc69da6688d86(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:27,265 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:27,265 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/B, priority=13, startTime=1732504466745; duration=0sec 2024-11-25T03:14:27,265 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:27,265 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:B 2024-11-25T03:14:27,265 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:27,266 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:27,266 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/C is initiating minor compaction (all files) 2024-11-25T03:14:27,266 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/C in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:27,267 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/33ab486886df4e209f8148bc40c96aac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/8a646ef51a844b9f8b9af821c595cadd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/4818c87fd0f94d30a1daf4f02ce84db3] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=35.2 K 2024-11-25T03:14:27,268 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ab486886df4e209f8148bc40c96aac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732504464068 2024-11-25T03:14:27,269 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a646ef51a844b9f8b9af821c595cadd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732504464206 2024-11-25T03:14:27,270 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4818c87fd0f94d30a1daf4f02ce84db3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732504464571 2024-11-25T03:14:27,303 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:27,304 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/54401ca2d6bf40c087edc40caf03547b is 50, key is test_row_0/C:col10/1732504464571/Put/seqid=0 2024-11-25T03:14:27,315 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/b9a9b6d6b2564ef288731a36d81a2172 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9a9b6d6b2564ef288731a36d81a2172 2024-11-25T03:14:27,325 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/A of 73fa306731753442713ff2a6a1c4403b into b9a9b6d6b2564ef288731a36d81a2172(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:27,325 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:27,325 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/A, priority=13, startTime=1732504466738; duration=0sec 2024-11-25T03:14:27,325 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:27,325 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:A 2024-11-25T03:14:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:14:27,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741976_1152 (size=12104) 2024-11-25T03:14:27,358 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/54401ca2d6bf40c087edc40caf03547b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/54401ca2d6bf40c087edc40caf03547b 2024-11-25T03:14:27,365 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/C of 73fa306731753442713ff2a6a1c4403b into 54401ca2d6bf40c087edc40caf03547b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:27,365 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:27,365 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/C, priority=13, startTime=1732504466745; duration=0sec 2024-11-25T03:14:27,365 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:27,365 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:C 2024-11-25T03:14:27,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:27,392 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112543c6ba20653a40e184b12870b28d4a21_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112543c6ba20653a40e184b12870b28d4a21_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/6b4e6831e1f5424ca6e7e2268bb25760, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:27,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/6b4e6831e1f5424ca6e7e2268bb25760 is 175, key is test_row_0/A:col10/1732504465271/Put/seqid=0 2024-11-25T03:14:27,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741977_1153 (size=30955) 2024-11-25T03:14:27,415 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/6b4e6831e1f5424ca6e7e2268bb25760 2024-11-25T03:14:27,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1 is 50, key is test_row_0/B:col10/1732504465271/Put/seqid=0 2024-11-25T03:14:27,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741978_1154 (size=12001) 2024-11-25T03:14:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:27,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504527465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504527469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504527469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504527471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504527580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504527580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504527590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504527591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504527791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504527791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504527800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504527810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:27,841 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1 2024-11-25T03:14:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/5dcf993de6ef4838823219bb40eadfde is 50, key is test_row_0/C:col10/1732504465271/Put/seqid=0 2024-11-25T03:14:27,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741979_1155 (size=12001) 2024-11-25T03:14:27,918 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/5dcf993de6ef4838823219bb40eadfde 2024-11-25T03:14:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/6b4e6831e1f5424ca6e7e2268bb25760 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/6b4e6831e1f5424ca6e7e2268bb25760 2024-11-25T03:14:27,948 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/6b4e6831e1f5424ca6e7e2268bb25760, entries=150, sequenceid=80, filesize=30.2 K 2024-11-25T03:14:27,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1 2024-11-25T03:14:27,958 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-25T03:14:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/5dcf993de6ef4838823219bb40eadfde as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/5dcf993de6ef4838823219bb40eadfde 2024-11-25T03:14:27,965 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/5dcf993de6ef4838823219bb40eadfde, entries=150, sequenceid=80, filesize=11.7 K 2024-11-25T03:14:27,967 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 73fa306731753442713ff2a6a1c4403b in 1078ms, sequenceid=80, compaction requested=false 2024-11-25T03:14:27,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:27,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:27,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-25T03:14:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-25T03:14:27,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-25T03:14:27,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7850 sec 2024-11-25T03:14:27,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.7950 sec 2024-11-25T03:14:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:28,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:14:28,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:28,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:28,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:28,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:28,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:28,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:28,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112564b337ac2bcd4353bdd1e1e4ee5e075e_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504468101/Put/seqid=0 2024-11-25T03:14:28,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504528154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504528155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504528162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504528163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741980_1156 (size=14594) 2024-11-25T03:14:28,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504528270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504528270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504528280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504528280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,295 INFO [master/38a5d2077268:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T03:14:28,295 INFO [master/38a5d2077268:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T03:14:28,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504528440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,441 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 73fa306731753442713ff2a6a1c4403b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504528490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504528490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504528500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504528500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:28,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112564b337ac2bcd4353bdd1e1e4ee5e075e_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112564b337ac2bcd4353bdd1e1e4ee5e075e_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:28,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/572344752f72463e8799d64aa0734f1b, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:28,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/572344752f72463e8799d64aa0734f1b is 175, key is test_row_0/A:col10/1732504468101/Put/seqid=0 2024-11-25T03:14:28,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741981_1157 (size=39549) 2024-11-25T03:14:28,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504528801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504528803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504528810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:28,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504528820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:29,012 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/572344752f72463e8799d64aa0734f1b 2024-11-25T03:14:29,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/339c03e1f3fe4ba1be1051b1fad01480 is 50, key is test_row_0/B:col10/1732504468101/Put/seqid=0 2024-11-25T03:14:29,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741982_1158 (size=12001) 2024-11-25T03:14:29,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/339c03e1f3fe4ba1be1051b1fad01480 2024-11-25T03:14:29,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/b3550586975b45388da62c98ed31e436 is 50, key is test_row_0/C:col10/1732504468101/Put/seqid=0 2024-11-25T03:14:29,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741983_1159 (size=12001) 2024-11-25T03:14:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504529310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504529310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504529320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:29,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:14:29,340 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:29,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-25T03:14:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:14:29,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504529340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:29,346 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:29,349 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:29,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:14:29,458 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:14:29,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T03:14:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:29,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:29,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:29,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:29,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:29,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/b3550586975b45388da62c98ed31e436 2024-11-25T03:14:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/572344752f72463e8799d64aa0734f1b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/572344752f72463e8799d64aa0734f1b 2024-11-25T03:14:29,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/572344752f72463e8799d64aa0734f1b, entries=200, sequenceid=98, filesize=38.6 K 2024-11-25T03:14:29,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/339c03e1f3fe4ba1be1051b1fad01480 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/339c03e1f3fe4ba1be1051b1fad01480 2024-11-25T03:14:29,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/339c03e1f3fe4ba1be1051b1fad01480, entries=150, sequenceid=98, filesize=11.7 K 2024-11-25T03:14:29,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/b3550586975b45388da62c98ed31e436 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b3550586975b45388da62c98ed31e436 2024-11-25T03:14:29,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b3550586975b45388da62c98ed31e436, entries=150, sequenceid=98, filesize=11.7 K 2024-11-25T03:14:29,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 73fa306731753442713ff2a6a1c4403b in 1492ms, sequenceid=98, compaction requested=true 2024-11-25T03:14:29,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:29,596 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:29,597 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:29,597 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/A is initiating minor compaction (all files) 2024-11-25T03:14:29,597 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/A in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:29,597 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9a9b6d6b2564ef288731a36d81a2172, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/6b4e6831e1f5424ca6e7e2268bb25760, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/572344752f72463e8799d64aa0734f1b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=99.2 K 2024-11-25T03:14:29,597 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:29,597 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9a9b6d6b2564ef288731a36d81a2172, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/6b4e6831e1f5424ca6e7e2268bb25760, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/572344752f72463e8799d64aa0734f1b] 2024-11-25T03:14:29,598 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9a9b6d6b2564ef288731a36d81a2172, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732504464571 2024-11-25T03:14:29,598 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b4e6831e1f5424ca6e7e2268bb25760, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732504465256 2024-11-25T03:14:29,599 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572344752f72463e8799d64aa0734f1b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732504467456 2024-11-25T03:14:29,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:29,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:29,605 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:29,606 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:29,607 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/B is initiating minor compaction (all files) 2024-11-25T03:14:29,607 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/B in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:29,607 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef25de7fcee840bdb1fcc69da6688d86, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/339c03e1f3fe4ba1be1051b1fad01480] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=35.3 K 2024-11-25T03:14:29,607 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ef25de7fcee840bdb1fcc69da6688d86, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732504464571 2024-11-25T03:14:29,608 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting fcffd4d98fd14d6da52ac4c5ea1b4fc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732504465256 2024-11-25T03:14:29,608 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 339c03e1f3fe4ba1be1051b1fad01480, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732504467456 2024-11-25T03:14:29,613 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:29,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:29,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:29,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:29,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:29,619 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411253ea349101684427cb249319608182d71_73fa306731753442713ff2a6a1c4403b store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:29,622 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411253ea349101684427cb249319608182d71_73fa306731753442713ff2a6a1c4403b, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:29,623 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253ea349101684427cb249319608182d71_73fa306731753442713ff2a6a1c4403b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:29,626 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:29,627 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5f901af069a74fd8811ff8f0d5ff2f44 is 50, key is test_row_0/B:col10/1732504468101/Put/seqid=0 2024-11-25T03:14:29,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741984_1160 (size=4469) 2024-11-25T03:14:29,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741985_1161 (size=12207) 2024-11-25T03:14:29,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T03:14:29,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:29,658 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:14:29,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:14:29,664 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5f901af069a74fd8811ff8f0d5ff2f44 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5f901af069a74fd8811ff8f0d5ff2f44 2024-11-25T03:14:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112568428c32cb314446a852b2b76e95935e_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504468151/Put/seqid=0 2024-11-25T03:14:29,678 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/B of 73fa306731753442713ff2a6a1c4403b into 5f901af069a74fd8811ff8f0d5ff2f44(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:29,678 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:29,678 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/B, priority=13, startTime=1732504469605; duration=0sec 2024-11-25T03:14:29,678 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:29,678 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:B 2024-11-25T03:14:29,678 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:29,680 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:29,680 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/C is initiating minor compaction (all files) 2024-11-25T03:14:29,680 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/C in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:29,680 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/54401ca2d6bf40c087edc40caf03547b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/5dcf993de6ef4838823219bb40eadfde, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b3550586975b45388da62c98ed31e436] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=35.3 K 2024-11-25T03:14:29,681 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 54401ca2d6bf40c087edc40caf03547b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732504464571 2024-11-25T03:14:29,682 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dcf993de6ef4838823219bb40eadfde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732504465256 2024-11-25T03:14:29,682 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b3550586975b45388da62c98ed31e436, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732504467456 2024-11-25T03:14:29,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741986_1162 (size=12154) 2024-11-25T03:14:29,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:29,691 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112568428c32cb314446a852b2b76e95935e_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112568428c32cb314446a852b2b76e95935e_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/34e0d3a884cb4fa49caecb6f0abbcdac, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/34e0d3a884cb4fa49caecb6f0abbcdac is 175, key is test_row_0/A:col10/1732504468151/Put/seqid=0 2024-11-25T03:14:29,711 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:29,712 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/05b57687e7d946289d7ee013287e0824 is 50, key is test_row_0/C:col10/1732504468101/Put/seqid=0 2024-11-25T03:14:29,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741987_1163 (size=30955) 2024-11-25T03:14:29,717 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/34e0d3a884cb4fa49caecb6f0abbcdac 2024-11-25T03:14:29,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741988_1164 (size=12207) 2024-11-25T03:14:29,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/a3d258d5bdc745b3961e2096cb8f594d is 50, key is test_row_0/B:col10/1732504468151/Put/seqid=0 2024-11-25T03:14:29,739 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/05b57687e7d946289d7ee013287e0824 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/05b57687e7d946289d7ee013287e0824 2024-11-25T03:14:29,747 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/C of 73fa306731753442713ff2a6a1c4403b into 05b57687e7d946289d7ee013287e0824(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:29,747 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:29,747 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/C, priority=13, startTime=1732504469618; duration=0sec 2024-11-25T03:14:29,747 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:29,747 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:C 2024-11-25T03:14:29,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741989_1165 (size=12001) 2024-11-25T03:14:29,766 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/a3d258d5bdc745b3961e2096cb8f594d 2024-11-25T03:14:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/3c1bafd690d245ba86e78ce8d7144bdc is 50, key is test_row_0/C:col10/1732504468151/Put/seqid=0 2024-11-25T03:14:29,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741990_1166 (size=12001) 2024-11-25T03:14:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:14:30,048 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#A#compaction#138 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:30,049 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/07c941c1e91943deb826a37c0eef4457 is 175, key is test_row_0/A:col10/1732504468101/Put/seqid=0 2024-11-25T03:14:30,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741991_1167 (size=31161) 2024-11-25T03:14:30,066 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/07c941c1e91943deb826a37c0eef4457 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/07c941c1e91943deb826a37c0eef4457 2024-11-25T03:14:30,078 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/A of 73fa306731753442713ff2a6a1c4403b into 07c941c1e91943deb826a37c0eef4457(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:30,078 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:30,078 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/A, priority=13, startTime=1732504469596; duration=0sec 2024-11-25T03:14:30,078 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:30,078 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:A 2024-11-25T03:14:30,200 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/3c1bafd690d245ba86e78ce8d7144bdc 2024-11-25T03:14:30,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/34e0d3a884cb4fa49caecb6f0abbcdac as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/34e0d3a884cb4fa49caecb6f0abbcdac 2024-11-25T03:14:30,223 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/34e0d3a884cb4fa49caecb6f0abbcdac, entries=150, sequenceid=115, filesize=30.2 K 2024-11-25T03:14:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/a3d258d5bdc745b3961e2096cb8f594d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/a3d258d5bdc745b3961e2096cb8f594d 2024-11-25T03:14:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,231 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/a3d258d5bdc745b3961e2096cb8f594d, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T03:14:30,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/3c1bafd690d245ba86e78ce8d7144bdc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c1bafd690d245ba86e78ce8d7144bdc 2024-11-25T03:14:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,240 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c1bafd690d245ba86e78ce8d7144bdc, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T03:14:30,241 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 73fa306731753442713ff2a6a1c4403b in 583ms, sequenceid=115, compaction requested=false 2024-11-25T03:14:30,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:30,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:30,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-25T03:14:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-25T03:14:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-25T03:14:30,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 895 msec 2024-11-25T03:14:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 906 msec 2024-11-25T03:14:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:14:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:30,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:30,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:30,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:30,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:30,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:30,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:30,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125256b4cfe9c204b42a3cabc7bc97a6521_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504470433/Put/seqid=0 2024-11-25T03:14:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:14:30,480 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:30,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-25T03:14:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:14:30,484 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:30,485 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:30,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:30,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504470497, max timestamp=1732504470511Num uniques 859; 2024-11-25T03:14:30,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:14:30,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504470497, max timestamp=1732504470511Num uniques 859; 2024-11-25T03:14:30,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:14:30,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504470497, max timestamp=1732504470508Num uniques 859; 2024-11-25T03:14:30,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:14:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:14:30,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741993_1169 (size=24408) 2024-11-25T03:14:30,595 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:30,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504530548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125256b4cfe9c204b42a3cabc7bc97a6521_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125256b4cfe9c204b42a3cabc7bc97a6521_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:30,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/0bcc33e8ed6f41889195a45a326f211c, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:30,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/0bcc33e8ed6f41889195a45a326f211c is 175, key is test_row_0/A:col10/1732504470433/Put/seqid=0 2024-11-25T03:14:30,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504530599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741992_1168 (size=74045) 2024-11-25T03:14:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504530602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504530605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:30,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504530710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504530720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504530720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504530720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:30,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:14:30,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504530920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504530940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504530940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504530940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:30,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:30,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,015 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/0bcc33e8ed6f41889195a45a326f211c 2024-11-25T03:14:31,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/f991ecfcd421432b91a4ce484624445d is 50, key is test_row_0/B:col10/1732504470433/Put/seqid=0 2024-11-25T03:14:31,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741994_1170 (size=12051) 2024-11-25T03:14:31,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/f991ecfcd421432b91a4ce484624445d 2024-11-25T03:14:31,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/da527e11f30b4fa3ba3e61635ef622e6 is 50, key is test_row_0/C:col10/1732504470433/Put/seqid=0 2024-11-25T03:14:31,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741995_1171 (size=12051) 2024-11-25T03:14:31,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/da527e11f30b4fa3ba3e61635ef622e6 2024-11-25T03:14:31,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:31,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:31,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:14:31,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/0bcc33e8ed6f41889195a45a326f211c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/0bcc33e8ed6f41889195a45a326f211c 2024-11-25T03:14:31,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/0bcc33e8ed6f41889195a45a326f211c, entries=400, sequenceid=129, filesize=72.3 K 2024-11-25T03:14:31,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/f991ecfcd421432b91a4ce484624445d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f991ecfcd421432b91a4ce484624445d 2024-11-25T03:14:31,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f991ecfcd421432b91a4ce484624445d, entries=150, sequenceid=129, filesize=11.8 K 2024-11-25T03:14:31,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/da527e11f30b4fa3ba3e61635ef622e6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/da527e11f30b4fa3ba3e61635ef622e6 2024-11-25T03:14:31,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/da527e11f30b4fa3ba3e61635ef622e6, entries=150, sequenceid=129, filesize=11.8 K 2024-11-25T03:14:31,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 73fa306731753442713ff2a6a1c4403b in 696ms, sequenceid=129, compaction requested=true 2024-11-25T03:14:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:31,145 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:31,145 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:31,147 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:31,147 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/A is initiating minor compaction (all files) 2024-11-25T03:14:31,147 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/A in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,147 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/07c941c1e91943deb826a37c0eef4457, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/34e0d3a884cb4fa49caecb6f0abbcdac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/0bcc33e8ed6f41889195a45a326f211c] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=133.0 K 2024-11-25T03:14:31,147 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,147 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/07c941c1e91943deb826a37c0eef4457, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/34e0d3a884cb4fa49caecb6f0abbcdac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/0bcc33e8ed6f41889195a45a326f211c] 2024-11-25T03:14:31,148 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:31,148 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/B is initiating minor compaction (all files) 2024-11-25T03:14:31,148 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/B in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,149 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5f901af069a74fd8811ff8f0d5ff2f44, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/a3d258d5bdc745b3961e2096cb8f594d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f991ecfcd421432b91a4ce484624445d] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=35.4 K 2024-11-25T03:14:31,149 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07c941c1e91943deb826a37c0eef4457, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732504467456 2024-11-25T03:14:31,150 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34e0d3a884cb4fa49caecb6f0abbcdac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504468151 2024-11-25T03:14:31,150 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f901af069a74fd8811ff8f0d5ff2f44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732504467456 2024-11-25T03:14:31,150 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d258d5bdc745b3961e2096cb8f594d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504468151 2024-11-25T03:14:31,151 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcc33e8ed6f41889195a45a326f211c, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732504470433 2024-11-25T03:14:31,151 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f991ecfcd421432b91a4ce484624445d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732504470433 2024-11-25T03:14:31,169 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:31,172 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:31,172 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411255fffd811247145d88161ec837c19e264_73fa306731753442713ff2a6a1c4403b store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:31,175 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411255fffd811247145d88161ec837c19e264_73fa306731753442713ff2a6a1c4403b, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:31,175 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255fffd811247145d88161ec837c19e264_73fa306731753442713ff2a6a1c4403b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:31,177 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/b080aee9db334155a7a31242e43e7fcc is 50, key is test_row_0/B:col10/1732504470433/Put/seqid=0 2024-11-25T03:14:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741997_1173 (size=12359) 2024-11-25T03:14:31,201 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/b080aee9db334155a7a31242e43e7fcc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b080aee9db334155a7a31242e43e7fcc 2024-11-25T03:14:31,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741996_1172 (size=4469) 2024-11-25T03:14:31,211 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/B of 73fa306731753442713ff2a6a1c4403b into b080aee9db334155a7a31242e43e7fcc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:31,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:31,211 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/B, priority=13, startTime=1732504471145; duration=0sec 2024-11-25T03:14:31,211 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#A#compaction#147 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:31,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:31,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:B 2024-11-25T03:14:31,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:31,212 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/339d8693797a4cf7ae5a04bfb004bfd9 is 175, key is test_row_0/A:col10/1732504470433/Put/seqid=0 2024-11-25T03:14:31,217 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:31,218 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/C is initiating minor compaction (all files) 2024-11-25T03:14:31,218 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/C in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,218 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/05b57687e7d946289d7ee013287e0824, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c1bafd690d245ba86e78ce8d7144bdc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/da527e11f30b4fa3ba3e61635ef622e6] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=35.4 K 2024-11-25T03:14:31,219 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 05b57687e7d946289d7ee013287e0824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732504467456 2024-11-25T03:14:31,220 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c1bafd690d245ba86e78ce8d7144bdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504468151 2024-11-25T03:14:31,221 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting da527e11f30b4fa3ba3e61635ef622e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732504470433 2024-11-25T03:14:31,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741998_1174 (size=31313) 2024-11-25T03:14:31,246 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/339d8693797a4cf7ae5a04bfb004bfd9 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/339d8693797a4cf7ae5a04bfb004bfd9 2024-11-25T03:14:31,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-25T03:14:31,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:31,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:31,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:31,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:31,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:31,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:31,257 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/A of 73fa306731753442713ff2a6a1c4403b into 339d8693797a4cf7ae5a04bfb004bfd9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:31,257 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:31,257 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/A, priority=13, startTime=1732504471145; duration=0sec 2024-11-25T03:14:31,257 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:31,257 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:A 2024-11-25T03:14:31,258 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#C#compaction#149 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:31,258 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/ca54304f350747d790f5e8ae58b98150 is 50, key is test_row_0/C:col10/1732504470433/Put/seqid=0 2024-11-25T03:14:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:31,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:31,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:31,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504531276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504531279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504531281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504531281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c78c993568ee4a1f9736f8887f0bb2c3_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504471243/Put/seqid=0 2024-11-25T03:14:31,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741999_1175 (size=12359) 2024-11-25T03:14:31,340 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/ca54304f350747d790f5e8ae58b98150 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/ca54304f350747d790f5e8ae58b98150 2024-11-25T03:14:31,349 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/C of 73fa306731753442713ff2a6a1c4403b into ca54304f350747d790f5e8ae58b98150(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:31,349 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:31,349 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/C, priority=13, startTime=1732504471145; duration=0sec 2024-11-25T03:14:31,349 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:31,349 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:C 2024-11-25T03:14:31,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742000_1176 (size=12304) 2024-11-25T03:14:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504531390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504531400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504531400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504531401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:31,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:31,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504531601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504531610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504531610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504531610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:14:31,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,750 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:31,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c78c993568ee4a1f9736f8887f0bb2c3_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c78c993568ee4a1f9736f8887f0bb2c3_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:31,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c0563dae45eb4362bac448c0873c1a0e, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:31,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c0563dae45eb4362bac448c0873c1a0e is 175, key is test_row_0/A:col10/1732504471243/Put/seqid=0 2024-11-25T03:14:31,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742001_1177 (size=31105) 2024-11-25T03:14:31,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:31,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504531920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504531920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504531920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:31,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:31,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504531930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:32,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:32,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,167 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c0563dae45eb4362bac448c0873c1a0e 2024-11-25T03:14:32,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/ef2e514affe74c76b66126a223ac375d is 50, key is test_row_0/B:col10/1732504471243/Put/seqid=0 2024-11-25T03:14:32,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742002_1178 (size=12151) 2024-11-25T03:14:32,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/ef2e514affe74c76b66126a223ac375d 2024-11-25T03:14:32,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/802ae0e31c87401cab86aa8f50d5804b is 50, key is test_row_0/C:col10/1732504471243/Put/seqid=0 2024-11-25T03:14:32,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742003_1179 (size=12151) 2024-11-25T03:14:32,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:32,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504532430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:32,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504532440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:32,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:32,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:32,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504532440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504532440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:32,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504532489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:32,494 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 73fa306731753442713ff2a6a1c4403b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:32,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:14:32,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/802ae0e31c87401cab86aa8f50d5804b 2024-11-25T03:14:32,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:32,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:32,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c0563dae45eb4362bac448c0873c1a0e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c0563dae45eb4362bac448c0873c1a0e 2024-11-25T03:14:32,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c0563dae45eb4362bac448c0873c1a0e, entries=150, sequenceid=167, filesize=30.4 K 2024-11-25T03:14:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/ef2e514affe74c76b66126a223ac375d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef2e514affe74c76b66126a223ac375d 2024-11-25T03:14:32,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef2e514affe74c76b66126a223ac375d, entries=150, sequenceid=167, filesize=11.9 K 2024-11-25T03:14:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/802ae0e31c87401cab86aa8f50d5804b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/802ae0e31c87401cab86aa8f50d5804b 2024-11-25T03:14:32,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/802ae0e31c87401cab86aa8f50d5804b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-25T03:14:32,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=46.96 KB/48090 for 73fa306731753442713ff2a6a1c4403b in 1431ms, sequenceid=167, compaction requested=false 2024-11-25T03:14:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:32,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:14:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:32,814 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:14:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252482868054ca4056bfa8eb511670347c_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504471277/Put/seqid=0 2024-11-25T03:14:32,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742004_1180 (size=12304) 2024-11-25T03:14:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:32,868 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252482868054ca4056bfa8eb511670347c_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252482868054ca4056bfa8eb511670347c_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/d7497abf4e66476ca3a1ca75febb41fe, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/d7497abf4e66476ca3a1ca75febb41fe is 175, key is test_row_0/A:col10/1732504471277/Put/seqid=0 2024-11-25T03:14:32,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742005_1181 (size=31105) 2024-11-25T03:14:33,302 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/d7497abf4e66476ca3a1ca75febb41fe 2024-11-25T03:14:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/f00f70e7305b4b8eaf8a36abaa2502af is 50, key is test_row_0/B:col10/1732504471277/Put/seqid=0 2024-11-25T03:14:33,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742006_1182 (size=12151) 2024-11-25T03:14:33,337 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/f00f70e7305b4b8eaf8a36abaa2502af 2024-11-25T03:14:33,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/36fe1e2bee9141a5ba0b0b511cd80bf0 is 50, key is test_row_0/C:col10/1732504471277/Put/seqid=0 2024-11-25T03:14:33,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742007_1183 (size=12151) 2024-11-25T03:14:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:33,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:33,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732504473470, max timestamp=1732504473486Num uniques 859; 2024-11-25T03:14:33,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732504473470, max timestamp=1732504473486Num uniques 859; 2024-11-25T03:14:33,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T03:14:33,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T03:14:33,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732504473470, max timestamp=1732504473486Num uniques 859; 2024-11-25T03:14:33,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T03:14:33,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504533518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504533521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504533522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504533524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504533630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504533630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504533640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504533640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,835 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/36fe1e2bee9141a5ba0b0b511cd80bf0 2024-11-25T03:14:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504533840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504533840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/d7497abf4e66476ca3a1ca75febb41fe as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/d7497abf4e66476ca3a1ca75febb41fe 2024-11-25T03:14:33,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504533850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,851 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/d7497abf4e66476ca3a1ca75febb41fe, entries=150, sequenceid=179, filesize=30.4 K 2024-11-25T03:14:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504533851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/f00f70e7305b4b8eaf8a36abaa2502af as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f00f70e7305b4b8eaf8a36abaa2502af 2024-11-25T03:14:33,858 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f00f70e7305b4b8eaf8a36abaa2502af, entries=150, sequenceid=179, filesize=11.9 K 2024-11-25T03:14:33,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/36fe1e2bee9141a5ba0b0b511cd80bf0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/36fe1e2bee9141a5ba0b0b511cd80bf0 2024-11-25T03:14:33,864 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/36fe1e2bee9141a5ba0b0b511cd80bf0, entries=150, sequenceid=179, filesize=11.9 K 2024-11-25T03:14:33,866 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 73fa306731753442713ff2a6a1c4403b in 1052ms, sequenceid=179, compaction requested=true 2024-11-25T03:14:33,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:33,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:33,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-25T03:14:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-25T03:14:33,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-25T03:14:33,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3820 sec 2024-11-25T03:14:33,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.3890 sec 2024-11-25T03:14:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:34,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-25T03:14:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:34,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d7b92ef6bb1b4301bb398486bc91f975_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504474155/Put/seqid=0 2024-11-25T03:14:34,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742008_1184 (size=17284) 2024-11-25T03:14:34,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504534165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d7b92ef6bb1b4301bb398486bc91f975_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d7b92ef6bb1b4301bb398486bc91f975_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:34,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/612b7c5cae294059b01490aeda61e8c3, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:34,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/612b7c5cae294059b01490aeda61e8c3 is 175, key is test_row_0/A:col10/1732504474155/Put/seqid=0 2024-11-25T03:14:34,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504534174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504534174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504534177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742009_1185 (size=48389) 2024-11-25T03:14:34,216 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/612b7c5cae294059b01490aeda61e8c3 2024-11-25T03:14:34,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5edc2df839654612b48b9dab5d422f73 is 50, key is test_row_0/B:col10/1732504474155/Put/seqid=0 2024-11-25T03:14:34,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742010_1186 (size=12151) 2024-11-25T03:14:34,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5edc2df839654612b48b9dab5d422f73 2024-11-25T03:14:34,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/eafe3618084b4d43a17fa49d9eed0267 is 50, key is test_row_0/C:col10/1732504474155/Put/seqid=0 2024-11-25T03:14:34,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742011_1187 (size=12151) 2024-11-25T03:14:34,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/eafe3618084b4d43a17fa49d9eed0267 2024-11-25T03:14:34,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504534280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/612b7c5cae294059b01490aeda61e8c3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/612b7c5cae294059b01490aeda61e8c3 2024-11-25T03:14:34,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504534290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504534290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/612b7c5cae294059b01490aeda61e8c3, entries=250, sequenceid=216, filesize=47.3 K 2024-11-25T03:14:34,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5edc2df839654612b48b9dab5d422f73 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5edc2df839654612b48b9dab5d422f73 2024-11-25T03:14:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5edc2df839654612b48b9dab5d422f73, entries=150, sequenceid=216, filesize=11.9 K 2024-11-25T03:14:34,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/eafe3618084b4d43a17fa49d9eed0267 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/eafe3618084b4d43a17fa49d9eed0267 2024-11-25T03:14:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504534300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/eafe3618084b4d43a17fa49d9eed0267, entries=150, sequenceid=216, filesize=11.9 K 2024-11-25T03:14:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 73fa306731753442713ff2a6a1c4403b in 155ms, sequenceid=216, compaction requested=true 2024-11-25T03:14:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:34,311 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:34,311 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:34,313 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:34,313 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:34,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:34,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:34,314 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/A is initiating minor compaction (all files) 2024-11-25T03:14:34,314 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/B is initiating minor compaction (all files) 2024-11-25T03:14:34,314 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/B in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,314 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/A in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,315 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/339d8693797a4cf7ae5a04bfb004bfd9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c0563dae45eb4362bac448c0873c1a0e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/d7497abf4e66476ca3a1ca75febb41fe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/612b7c5cae294059b01490aeda61e8c3] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=138.6 K 2024-11-25T03:14:34,315 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b080aee9db334155a7a31242e43e7fcc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef2e514affe74c76b66126a223ac375d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f00f70e7305b4b8eaf8a36abaa2502af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5edc2df839654612b48b9dab5d422f73] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=47.7 K 2024-11-25T03:14:34,315 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,315 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/339d8693797a4cf7ae5a04bfb004bfd9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c0563dae45eb4362bac448c0873c1a0e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/d7497abf4e66476ca3a1ca75febb41fe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/612b7c5cae294059b01490aeda61e8c3] 2024-11-25T03:14:34,315 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b080aee9db334155a7a31242e43e7fcc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732504470433 2024-11-25T03:14:34,315 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339d8693797a4cf7ae5a04bfb004bfd9, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732504470433 2024-11-25T03:14:34,315 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2e514affe74c76b66126a223ac375d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732504470525 2024-11-25T03:14:34,316 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0563dae45eb4362bac448c0873c1a0e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732504470525 2024-11-25T03:14:34,316 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f00f70e7305b4b8eaf8a36abaa2502af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732504471276 2024-11-25T03:14:34,316 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7497abf4e66476ca3a1ca75febb41fe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732504471276 2024-11-25T03:14:34,316 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edc2df839654612b48b9dab5d422f73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732504473522 2024-11-25T03:14:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,317 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 612b7c5cae294059b01490aeda61e8c3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732504473511 2024-11-25T03:14:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,329 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,354 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125c94230b40f5546389b5fdc05248971de_73fa306731753442713ff2a6a1c4403b store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:34,354 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#B#compaction#160 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,355 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/eb561894d6bb4171802d78c446969f31 is 50, key is test_row_0/B:col10/1732504474155/Put/seqid=0 2024-11-25T03:14:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,360 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125c94230b40f5546389b5fdc05248971de_73fa306731753442713ff2a6a1c4403b, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:34,360 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c94230b40f5546389b5fdc05248971de_73fa306731753442713ff2a6a1c4403b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742013_1189 (size=4469) 2024-11-25T03:14:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742012_1188 (size=12595) 2024-11-25T03:14:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,392 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/eb561894d6bb4171802d78c446969f31 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/eb561894d6bb4171802d78c446969f31 2024-11-25T03:14:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,401 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/B of 73fa306731753442713ff2a6a1c4403b into eb561894d6bb4171802d78c446969f31(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:34,401 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:34,401 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/B, priority=12, startTime=1732504474311; duration=0sec 2024-11-25T03:14:34,401 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:34,401 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:B 2024-11-25T03:14:34,401 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,403 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:34,403 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/C is initiating minor compaction (all files) 2024-11-25T03:14:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,403 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/C in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,404 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/ca54304f350747d790f5e8ae58b98150, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/802ae0e31c87401cab86aa8f50d5804b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/36fe1e2bee9141a5ba0b0b511cd80bf0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/eafe3618084b4d43a17fa49d9eed0267] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=47.7 K 2024-11-25T03:14:34,404 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ca54304f350747d790f5e8ae58b98150, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732504470433 2024-11-25T03:14:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,405 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 802ae0e31c87401cab86aa8f50d5804b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732504470525 2024-11-25T03:14:34,406 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 36fe1e2bee9141a5ba0b0b511cd80bf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732504471276 2024-11-25T03:14:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,406 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting eafe3618084b4d43a17fa49d9eed0267, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732504473522 2024-11-25T03:14:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,436 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,436 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/cb26280cd1594a9fa994a6a24ec81afe is 50, key is test_row_0/C:col10/1732504474155/Put/seqid=0 2024-11-25T03:14:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742014_1190 (size=12595) 2024-11-25T03:14:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,484 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/cb26280cd1594a9fa994a6a24ec81afe as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/cb26280cd1594a9fa994a6a24ec81afe 2024-11-25T03:14:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,492 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/C of 73fa306731753442713ff2a6a1c4403b into cb26280cd1594a9fa994a6a24ec81afe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:34,492 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:34,492 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/C, priority=12, startTime=1732504474314; duration=0sec 2024-11-25T03:14:34,492 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:34,492 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:C 2024-11-25T03:14:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:34,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:14:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112581e164bb27c14fbaa67c97a26f316150_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504474165/Put/seqid=0 2024-11-25T03:14:34,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742015_1191 (size=12304) 2024-11-25T03:14:34,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732504474537, max timestamp=1732504474555Num uniques 859; 2024-11-25T03:14:34,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732504474537, max timestamp=1732504474555Num uniques 859; 2024-11-25T03:14:34,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-25T03:14:34,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-25T03:14:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504534549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504534550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732504474537, max timestamp=1732504474555Num uniques 859; 2024-11-25T03:14:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-25T03:14:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:14:34,640 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:34,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-25T03:14:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:14:34,644 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:34,645 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:34,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504534642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504534644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504534670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504534670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:14:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504534760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504534760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,783 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#A#compaction#159 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:34,784 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/ae284665d3d8451cb26da5045b373da2 is 175, key is test_row_0/A:col10/1732504474155/Put/seqid=0 2024-11-25T03:14:34,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:14:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:34,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742016_1192 (size=31549) 2024-11-25T03:14:34,822 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/ae284665d3d8451cb26da5045b373da2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/ae284665d3d8451cb26da5045b373da2 2024-11-25T03:14:34,828 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/A of 73fa306731753442713ff2a6a1c4403b into ae284665d3d8451cb26da5045b373da2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:34,828 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:34,828 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/A, priority=12, startTime=1732504474311; duration=0sec 2024-11-25T03:14:34,828 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:34,829 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:A 2024-11-25T03:14:34,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504534881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504534881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:34,933 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112581e164bb27c14fbaa67c97a26f316150_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112581e164bb27c14fbaa67c97a26f316150_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:34,934 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/8433527e1f0f4e9fac93a0a459c493fb, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:34,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/8433527e1f0f4e9fac93a0a459c493fb is 175, key is test_row_0/A:col10/1732504474165/Put/seqid=0 2024-11-25T03:14:34,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742017_1193 (size=31105) 2024-11-25T03:14:34,947 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/8433527e1f0f4e9fac93a0a459c493fb 2024-11-25T03:14:34,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:14:34,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:34,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:34,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:34,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:14:34,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/2370006bd0264c5c937e8e353afe72ed is 50, key is test_row_0/B:col10/1732504474165/Put/seqid=0 2024-11-25T03:14:34,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742018_1194 (size=12151) 2024-11-25T03:14:34,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504534980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504534980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:14:35,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:35,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:35,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:35,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504535189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504535190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:35,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:14:35,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504535290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504535290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/2370006bd0264c5c937e8e353afe72ed 2024-11-25T03:14:35,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/e68760663a734edea3722d5825563aad is 50, key is test_row_0/C:col10/1732504474165/Put/seqid=0 2024-11-25T03:14:35,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742019_1195 (size=12151) 2024-11-25T03:14:35,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/e68760663a734edea3722d5825563aad 2024-11-25T03:14:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/8433527e1f0f4e9fac93a0a459c493fb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/8433527e1f0f4e9fac93a0a459c493fb 2024-11-25T03:14:35,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/8433527e1f0f4e9fac93a0a459c493fb, entries=150, sequenceid=228, filesize=30.4 K 2024-11-25T03:14:35,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/2370006bd0264c5c937e8e353afe72ed as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/2370006bd0264c5c937e8e353afe72ed 2024-11-25T03:14:35,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/2370006bd0264c5c937e8e353afe72ed, entries=150, sequenceid=228, filesize=11.9 K 2024-11-25T03:14:35,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/e68760663a734edea3722d5825563aad as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/e68760663a734edea3722d5825563aad 2024-11-25T03:14:35,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:14:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:35,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:35,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/e68760663a734edea3722d5825563aad, entries=150, sequenceid=228, filesize=11.9 K 2024-11-25T03:14:35,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 73fa306731753442713ff2a6a1c4403b in 916ms, sequenceid=228, compaction requested=false 2024-11-25T03:14:35,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:35,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:14:35,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:35,568 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-25T03:14:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:35,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256174f2e15fdf4c27a5bebcb221faca48_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504474641/Put/seqid=0 2024-11-25T03:14:35,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742020_1196 (size=12454) 2024-11-25T03:14:35,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:35,624 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256174f2e15fdf4c27a5bebcb221faca48_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256174f2e15fdf4c27a5bebcb221faca48_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/51ca08f67f2b448aa25d078ecf5ca548, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:35,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/51ca08f67f2b448aa25d078ecf5ca548 is 175, key is test_row_0/A:col10/1732504474641/Put/seqid=0 2024-11-25T03:14:35,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742021_1197 (size=31255) 2024-11-25T03:14:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:35,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504535720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504535732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:14:35,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504535801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504535801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504535840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:35,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504535850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,031 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/51ca08f67f2b448aa25d078ecf5ca548 2024-11-25T03:14:36,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/4f4f850e77b24ad6a1134477ff7b7c84 is 50, key is test_row_0/B:col10/1732504474641/Put/seqid=0 2024-11-25T03:14:36,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742022_1198 (size=12301) 2024-11-25T03:14:36,051 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/4f4f850e77b24ad6a1134477ff7b7c84 2024-11-25T03:14:36,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504536051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/2d4666763d7242a29e5316668a3ce348 is 50, key is test_row_0/C:col10/1732504474641/Put/seqid=0 2024-11-25T03:14:36,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504536063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742023_1199 (size=12301) 2024-11-25T03:14:36,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504536360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504536380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,472 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/2d4666763d7242a29e5316668a3ce348 2024-11-25T03:14:36,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/51ca08f67f2b448aa25d078ecf5ca548 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/51ca08f67f2b448aa25d078ecf5ca548 2024-11-25T03:14:36,481 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/51ca08f67f2b448aa25d078ecf5ca548, entries=150, sequenceid=263, filesize=30.5 K 2024-11-25T03:14:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/4f4f850e77b24ad6a1134477ff7b7c84 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/4f4f850e77b24ad6a1134477ff7b7c84 2024-11-25T03:14:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,486 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/4f4f850e77b24ad6a1134477ff7b7c84, entries=150, sequenceid=263, filesize=12.0 K 2024-11-25T03:14:36,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/2d4666763d7242a29e5316668a3ce348 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/2d4666763d7242a29e5316668a3ce348 2024-11-25T03:14:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,492 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/2d4666763d7242a29e5316668a3ce348, entries=150, sequenceid=263, filesize=12.0 K 2024-11-25T03:14:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,493 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 73fa306731753442713ff2a6a1c4403b in 925ms, sequenceid=263, compaction requested=true 2024-11-25T03:14:36,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:36,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:36,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-25T03:14:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-25T03:14:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-25T03:14:36,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-25T03:14:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8570 sec 2024-11-25T03:14:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:14:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,801 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-25T03:14:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:14:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,806 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:36,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:14:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:36,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:36,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:36,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:36,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259b7b04f32c1d4dcba9a569a81bb2319d_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504475728/Put/seqid=0 2024-11-25T03:14:36,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742024_1200 (size=12454) 2024-11-25T03:14:36,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:36,888 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259b7b04f32c1d4dcba9a569a81bb2319d_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259b7b04f32c1d4dcba9a569a81bb2319d_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:36,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/aeb65263cc5744238d9db0f8bfa4df6f, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:36,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/aeb65263cc5744238d9db0f8bfa4df6f is 175, key is test_row_0/A:col10/1732504475728/Put/seqid=0 2024-11-25T03:14:36,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742025_1201 (size=31255) 2024-11-25T03:14:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:14:36,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/aeb65263cc5744238d9db0f8bfa4df6f 2024-11-25T03:14:36,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504536901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504536901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732504476871, max timestamp=1732504476910Num uniques 859; 2024-11-25T03:14:36,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-25T03:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732504476871, max timestamp=1732504476910Num uniques 859; 2024-11-25T03:14:36,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-25T03:14:36,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732504476871, max timestamp=1732504476910Num uniques 859; 2024-11-25T03:14:36,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-25T03:14:36,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/be9fd7d01bdb426aa76ff6637cfe7105 is 50, key is test_row_0/B:col10/1732504475728/Put/seqid=0 2024-11-25T03:14:36,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504536947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:36,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:36,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504536956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:36,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742026_1202 (size=12301) 2024-11-25T03:14:37,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504537020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504537020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504537060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504537071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:37,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:37,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:14:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504537230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504537230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504537280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504537280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/be9fd7d01bdb426aa76ff6637cfe7105 2024-11-25T03:14:37,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/70bab88d80c2460ba61c78b33a086701 is 50, key is test_row_0/C:col10/1732504475728/Put/seqid=0 2024-11-25T03:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:14:37,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742027_1203 (size=12301) 2024-11-25T03:14:37,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/70bab88d80c2460ba61c78b33a086701 2024-11-25T03:14:37,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/aeb65263cc5744238d9db0f8bfa4df6f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/aeb65263cc5744238d9db0f8bfa4df6f 2024-11-25T03:14:37,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/aeb65263cc5744238d9db0f8bfa4df6f, entries=150, sequenceid=274, filesize=30.5 K 2024-11-25T03:14:37,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/be9fd7d01bdb426aa76ff6637cfe7105 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/be9fd7d01bdb426aa76ff6637cfe7105 2024-11-25T03:14:37,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/be9fd7d01bdb426aa76ff6637cfe7105, entries=150, sequenceid=274, filesize=12.0 K 2024-11-25T03:14:37,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/70bab88d80c2460ba61c78b33a086701 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/70bab88d80c2460ba61c78b33a086701 2024-11-25T03:14:37,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/70bab88d80c2460ba61c78b33a086701, entries=150, sequenceid=274, filesize=12.0 K 2024-11-25T03:14:37,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 73fa306731753442713ff2a6a1c4403b in 705ms, sequenceid=274, compaction requested=true 2024-11-25T03:14:37,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:37,534 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:37,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:37,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:37,534 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:37,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:37,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:37,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:37,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:37,538 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125164 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:37,538 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/A is initiating minor compaction (all files) 2024-11-25T03:14:37,538 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/A in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,539 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/ae284665d3d8451cb26da5045b373da2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/8433527e1f0f4e9fac93a0a459c493fb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/51ca08f67f2b448aa25d078ecf5ca548, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/aeb65263cc5744238d9db0f8bfa4df6f] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=122.2 K 2024-11-25T03:14:37,539 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,539 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/ae284665d3d8451cb26da5045b373da2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/8433527e1f0f4e9fac93a0a459c493fb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/51ca08f67f2b448aa25d078ecf5ca548, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/aeb65263cc5744238d9db0f8bfa4df6f] 2024-11-25T03:14:37,539 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:37,539 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae284665d3d8451cb26da5045b373da2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732504473522 2024-11-25T03:14:37,539 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/B is initiating minor compaction (all files) 2024-11-25T03:14:37,539 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/B in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,539 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/eb561894d6bb4171802d78c446969f31, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/2370006bd0264c5c937e8e353afe72ed, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/4f4f850e77b24ad6a1134477ff7b7c84, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/be9fd7d01bdb426aa76ff6637cfe7105] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=48.2 K 2024-11-25T03:14:37,540 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8433527e1f0f4e9fac93a0a459c493fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732504474165 2024-11-25T03:14:37,540 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting eb561894d6bb4171802d78c446969f31, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732504473522 2024-11-25T03:14:37,540 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51ca08f67f2b448aa25d078ecf5ca548, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732504474624 2024-11-25T03:14:37,541 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 2370006bd0264c5c937e8e353afe72ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732504474165 2024-11-25T03:14:37,541 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb65263cc5744238d9db0f8bfa4df6f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504475716 2024-11-25T03:14:37,541 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f4f850e77b24ad6a1134477ff7b7c84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732504474624 2024-11-25T03:14:37,542 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting be9fd7d01bdb426aa76ff6637cfe7105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504475716 2024-11-25T03:14:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:37,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-25T03:14:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:37,553 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:37,557 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125fdf5c3891afe476691270b24158a295f_73fa306731753442713ff2a6a1c4403b store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:37,559 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125fdf5c3891afe476691270b24158a295f_73fa306731753442713ff2a6a1c4403b, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:37,559 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fdf5c3891afe476691270b24158a295f_73fa306731753442713ff2a6a1c4403b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:37,566 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#B#compaction#172 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:37,566 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/236c1a786bfd40bd8baa5c3be4cab92c is 50, key is test_row_0/B:col10/1732504475728/Put/seqid=0 2024-11-25T03:14:37,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251d96316b1edf4664ba1cbc674fb1b076_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504476944/Put/seqid=0 2024-11-25T03:14:37,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504537580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504537582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504537590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504537590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742028_1204 (size=4469) 2024-11-25T03:14:37,612 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#A#compaction#171 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:37,612 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/33984fbe41d544dd988d0a3f9945e3ff is 175, key is test_row_0/A:col10/1732504475728/Put/seqid=0 2024-11-25T03:14:37,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742029_1205 (size=12881) 2024-11-25T03:14:37,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742030_1206 (size=14994) 2024-11-25T03:14:37,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742031_1207 (size=31835) 2024-11-25T03:14:37,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:37,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251d96316b1edf4664ba1cbc674fb1b076_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251d96316b1edf4664ba1cbc674fb1b076_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:37,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/780158e69516422fb02be48fc2f41e13, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:37,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/780158e69516422fb02be48fc2f41e13 is 175, key is test_row_0/A:col10/1732504476944/Put/seqid=0 2024-11-25T03:14:37,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742032_1208 (size=39949) 2024-11-25T03:14:37,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504537700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504537700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:37,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504537910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:37,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504537910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:14:38,031 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/236c1a786bfd40bd8baa5c3be4cab92c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/236c1a786bfd40bd8baa5c3be4cab92c 2024-11-25T03:14:38,040 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/B of 73fa306731753442713ff2a6a1c4403b into 236c1a786bfd40bd8baa5c3be4cab92c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:38,040 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:38,040 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/B, priority=12, startTime=1732504477534; duration=0sec 2024-11-25T03:14:38,041 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/33984fbe41d544dd988d0a3f9945e3ff as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/33984fbe41d544dd988d0a3f9945e3ff 2024-11-25T03:14:38,042 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:38,042 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:B 2024-11-25T03:14:38,043 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:38,046 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:38,046 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/C is initiating minor compaction (all files) 2024-11-25T03:14:38,046 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/C in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:38,046 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/cb26280cd1594a9fa994a6a24ec81afe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/e68760663a734edea3722d5825563aad, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/2d4666763d7242a29e5316668a3ce348, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/70bab88d80c2460ba61c78b33a086701] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=48.2 K 2024-11-25T03:14:38,048 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting cb26280cd1594a9fa994a6a24ec81afe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732504473522 2024-11-25T03:14:38,048 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/A of 73fa306731753442713ff2a6a1c4403b into 33984fbe41d544dd988d0a3f9945e3ff(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:38,048 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:38,048 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/A, priority=12, startTime=1732504477533; duration=0sec 2024-11-25T03:14:38,048 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:38,048 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:A 2024-11-25T03:14:38,049 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e68760663a734edea3722d5825563aad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732504474165 2024-11-25T03:14:38,051 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4666763d7242a29e5316668a3ce348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732504474624 2024-11-25T03:14:38,052 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 70bab88d80c2460ba61c78b33a086701, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504475716 2024-11-25T03:14:38,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/780158e69516422fb02be48fc2f41e13 2024-11-25T03:14:38,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:38,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:38,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:38,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:38,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:38,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:38,081 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:38,081 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/09419ffd50a44c1baf8f3d800d67dab0 is 50, key is test_row_0/C:col10/1732504475728/Put/seqid=0 2024-11-25T03:14:38,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/95ab79eefcea4ec880950df14bb1659b is 50, key is test_row_0/B:col10/1732504476944/Put/seqid=0 2024-11-25T03:14:38,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742033_1209 (size=12881) 2024-11-25T03:14:38,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504538100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504538110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742034_1210 (size=12301) 2024-11-25T03:14:38,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/95ab79eefcea4ec880950df14bb1659b 2024-11-25T03:14:38,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/a4c3b6ccc10d43b6a4123e4de5cc4a74 is 50, key is test_row_0/C:col10/1732504476944/Put/seqid=0 2024-11-25T03:14:38,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742035_1211 (size=12301) 2024-11-25T03:14:38,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/a4c3b6ccc10d43b6a4123e4de5cc4a74 2024-11-25T03:14:38,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/780158e69516422fb02be48fc2f41e13 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/780158e69516422fb02be48fc2f41e13 2024-11-25T03:14:38,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/780158e69516422fb02be48fc2f41e13, entries=200, sequenceid=307, filesize=39.0 K 2024-11-25T03:14:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/95ab79eefcea4ec880950df14bb1659b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/95ab79eefcea4ec880950df14bb1659b 2024-11-25T03:14:38,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/95ab79eefcea4ec880950df14bb1659b, entries=150, sequenceid=307, filesize=12.0 K 2024-11-25T03:14:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/a4c3b6ccc10d43b6a4123e4de5cc4a74 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a4c3b6ccc10d43b6a4123e4de5cc4a74 2024-11-25T03:14:38,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a4c3b6ccc10d43b6a4123e4de5cc4a74, entries=150, sequenceid=307, filesize=12.0 K 2024-11-25T03:14:38,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 73fa306731753442713ff2a6a1c4403b in 627ms, sequenceid=307, compaction requested=false 2024-11-25T03:14:38,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:38,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:14:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:38,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:38,234 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:14:38,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:38,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:38,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:38,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:38,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:38,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:38,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cc475db629f5454782fda330bb9a4138_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504478232/Put/seqid=0 2024-11-25T03:14:38,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742036_1212 (size=17534) 2024-11-25T03:14:38,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504478256, max timestamp=1732504478292Num uniques 859; 2024-11-25T03:14:38,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504478256, max timestamp=1732504478292Num uniques 859; 2024-11-25T03:14:38,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504478256, max timestamp=1732504478292Num uniques 859; 2024-11-25T03:14:38,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:14:38,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:14:38,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:14:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504538350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504538352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504538470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504538471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,498 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/09419ffd50a44c1baf8f3d800d67dab0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/09419ffd50a44c1baf8f3d800d67dab0 2024-11-25T03:14:38,503 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/C of 73fa306731753442713ff2a6a1c4403b into 09419ffd50a44c1baf8f3d800d67dab0(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:38,503 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:38,503 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/C, priority=12, startTime=1732504477534; duration=0sec 2024-11-25T03:14:38,503 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:38,503 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:C 2024-11-25T03:14:38,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:38,662 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cc475db629f5454782fda330bb9a4138_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cc475db629f5454782fda330bb9a4138_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/9e35f1c7f1f04f8682111624c680f3b3, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/9e35f1c7f1f04f8682111624c680f3b3 is 175, key is test_row_0/A:col10/1732504478232/Put/seqid=0 2024-11-25T03:14:38,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742037_1213 (size=48639) 2024-11-25T03:14:38,683 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/9e35f1c7f1f04f8682111624c680f3b3 2024-11-25T03:14:38,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504538680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504538680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:38,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/c046f31c936d42f9b2b9e8128dafffd7 is 50, key is test_row_0/B:col10/1732504478232/Put/seqid=0 2024-11-25T03:14:38,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742038_1214 (size=12301) 2024-11-25T03:14:38,728 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/c046f31c936d42f9b2b9e8128dafffd7 2024-11-25T03:14:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/b072e729f4054a87b8af81064a00d29f is 50, key is test_row_0/C:col10/1732504478232/Put/seqid=0 2024-11-25T03:14:38,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742039_1215 (size=12301) 2024-11-25T03:14:38,783 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/b072e729f4054a87b8af81064a00d29f 2024-11-25T03:14:38,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/9e35f1c7f1f04f8682111624c680f3b3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/9e35f1c7f1f04f8682111624c680f3b3 2024-11-25T03:14:38,806 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/9e35f1c7f1f04f8682111624c680f3b3, entries=250, sequenceid=321, filesize=47.5 K 2024-11-25T03:14:38,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/c046f31c936d42f9b2b9e8128dafffd7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/c046f31c936d42f9b2b9e8128dafffd7 2024-11-25T03:14:38,815 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/c046f31c936d42f9b2b9e8128dafffd7, entries=150, sequenceid=321, filesize=12.0 K 2024-11-25T03:14:38,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/b072e729f4054a87b8af81064a00d29f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b072e729f4054a87b8af81064a00d29f 2024-11-25T03:14:38,825 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b072e729f4054a87b8af81064a00d29f, entries=150, sequenceid=321, filesize=12.0 K 2024-11-25T03:14:38,827 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 73fa306731753442713ff2a6a1c4403b in 596ms, sequenceid=321, compaction requested=true 2024-11-25T03:14:38,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:38,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:38,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-25T03:14:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-25T03:14:38,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-25T03:14:38,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0230 sec 2024-11-25T03:14:38,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0310 sec 2024-11-25T03:14:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:14:38,971 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:38,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-25T03:14:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:14:38,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:38,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:38,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:38,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-25T03:14:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:39,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:39,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:39,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:39,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c0b63bd8b09b490d924409827d2df996_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504478350/Put/seqid=0 2024-11-25T03:14:39,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742040_1216 (size=14994) 2024-11-25T03:14:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504539053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504539063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:14:39,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504539121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504539122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,127 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 73fa306731753442713ff2a6a1c4403b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:39,127 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 73fa306731753442713ff2a6a1c4403b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:39,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:39,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504539170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504539180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:39,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:39,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:14:39,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504539381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504539400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:39,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c0b63bd8b09b490d924409827d2df996_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c0b63bd8b09b490d924409827d2df996_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:39,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/79cc678ee1e74db5b1db1915a6edd310, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:39,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/79cc678ee1e74db5b1db1915a6edd310 is 175, key is test_row_0/A:col10/1732504478350/Put/seqid=0 2024-11-25T03:14:39,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742041_1217 (size=39949) 2024-11-25T03:14:39,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:14:39,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504539700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504539710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:39,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:39,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:39,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,894 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/79cc678ee1e74db5b1db1915a6edd310 2024-11-25T03:14:39,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5e90d716ea7640bbb5f960e4cfd0efd4 is 50, key is test_row_0/B:col10/1732504478350/Put/seqid=0 2024-11-25T03:14:39,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:39,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:39,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742042_1218 (size=12301) 2024-11-25T03:14:40,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:14:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504540210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:40,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504540220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:40,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5e90d716ea7640bbb5f960e4cfd0efd4 2024-11-25T03:14:40,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/d1e3530627be4f8bb4a68cc003397d30 is 50, key is test_row_0/C:col10/1732504478350/Put/seqid=0 2024-11-25T03:14:40,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742043_1219 (size=12301) 2024-11-25T03:14:40,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:40,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:40,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:40,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:40,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:40,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/d1e3530627be4f8bb4a68cc003397d30 2024-11-25T03:14:40,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/79cc678ee1e74db5b1db1915a6edd310 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/79cc678ee1e74db5b1db1915a6edd310 2024-11-25T03:14:40,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/79cc678ee1e74db5b1db1915a6edd310, entries=200, sequenceid=357, filesize=39.0 K 2024-11-25T03:14:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/5e90d716ea7640bbb5f960e4cfd0efd4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5e90d716ea7640bbb5f960e4cfd0efd4 2024-11-25T03:14:40,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5e90d716ea7640bbb5f960e4cfd0efd4, entries=150, sequenceid=357, filesize=12.0 K 2024-11-25T03:14:40,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/d1e3530627be4f8bb4a68cc003397d30 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/d1e3530627be4f8bb4a68cc003397d30 2024-11-25T03:14:40,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/d1e3530627be4f8bb4a68cc003397d30, entries=150, sequenceid=357, filesize=12.0 K 2024-11-25T03:14:40,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 73fa306731753442713ff2a6a1c4403b in 1762ms, sequenceid=357, compaction requested=true 2024-11-25T03:14:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:40,761 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:40,761 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:40,763 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:40,763 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/A is initiating minor compaction (all files) 2024-11-25T03:14:40,763 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/A in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,763 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/33984fbe41d544dd988d0a3f9945e3ff, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/780158e69516422fb02be48fc2f41e13, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/9e35f1c7f1f04f8682111624c680f3b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/79cc678ee1e74db5b1db1915a6edd310] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=156.6 K 2024-11-25T03:14:40,763 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,763 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/33984fbe41d544dd988d0a3f9945e3ff, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/780158e69516422fb02be48fc2f41e13, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/9e35f1c7f1f04f8682111624c680f3b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/79cc678ee1e74db5b1db1915a6edd310] 2024-11-25T03:14:40,764 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:40,764 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/B is initiating minor compaction (all files) 2024-11-25T03:14:40,764 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33984fbe41d544dd988d0a3f9945e3ff, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504475716 2024-11-25T03:14:40,764 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/B in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,764 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/236c1a786bfd40bd8baa5c3be4cab92c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/95ab79eefcea4ec880950df14bb1659b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/c046f31c936d42f9b2b9e8128dafffd7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5e90d716ea7640bbb5f960e4cfd0efd4] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=48.6 K 2024-11-25T03:14:40,764 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 780158e69516422fb02be48fc2f41e13, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732504476944 2024-11-25T03:14:40,765 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 236c1a786bfd40bd8baa5c3be4cab92c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504475716 2024-11-25T03:14:40,765 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e35f1c7f1f04f8682111624c680f3b3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732504477567 2024-11-25T03:14:40,765 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ab79eefcea4ec880950df14bb1659b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732504476944 2024-11-25T03:14:40,765 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79cc678ee1e74db5b1db1915a6edd310, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732504478334 2024-11-25T03:14:40,766 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting c046f31c936d42f9b2b9e8128dafffd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732504477579 2024-11-25T03:14:40,766 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e90d716ea7640bbb5f960e4cfd0efd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732504478334 2024-11-25T03:14:40,774 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:40,776 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ab15d11d8bb44008b470f9fa0bd96e30_73fa306731753442713ff2a6a1c4403b store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:40,776 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:40,777 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/1225fa43e1f243f18c1062810a5e788e is 50, key is test_row_0/B:col10/1732504478350/Put/seqid=0 2024-11-25T03:14:40,778 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ab15d11d8bb44008b470f9fa0bd96e30_73fa306731753442713ff2a6a1c4403b, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:40,778 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ab15d11d8bb44008b470f9fa0bd96e30_73fa306731753442713ff2a6a1c4403b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:40,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742044_1220 (size=13017) 2024-11-25T03:14:40,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742045_1221 (size=4469) 2024-11-25T03:14:40,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:14:40,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:40,828 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:14:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:40,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d1733fe082a640f7a7257414753851c2_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504479060/Put/seqid=0 2024-11-25T03:14:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742046_1222 (size=12454) 2024-11-25T03:14:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:40,847 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d1733fe082a640f7a7257414753851c2_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d1733fe082a640f7a7257414753851c2_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:40,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/515ea4f458544d3fad756af054cc1f2c, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:40,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/515ea4f458544d3fad756af054cc1f2c is 175, key is test_row_0/A:col10/1732504479060/Put/seqid=0 2024-11-25T03:14:40,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742047_1223 (size=31255) 2024-11-25T03:14:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:14:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:41,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:41,198 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:41,199 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/20353384e0df45c08435e2683e37e3f6 is 175, key is test_row_0/A:col10/1732504478350/Put/seqid=0 2024-11-25T03:14:41,205 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/1225fa43e1f243f18c1062810a5e788e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/1225fa43e1f243f18c1062810a5e788e 2024-11-25T03:14:41,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742048_1224 (size=31971) 2024-11-25T03:14:41,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504481180, max timestamp=1732504481212Num uniques 859; 2024-11-25T03:14:41,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:14:41,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504481190, max timestamp=1732504481212Num uniques 859; 2024-11-25T03:14:41,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504481190, max timestamp=1732504481212Num uniques 859; 2024-11-25T03:14:41,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:41,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:41,222 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/B of 73fa306731753442713ff2a6a1c4403b into 1225fa43e1f243f18c1062810a5e788e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:41,222 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:41,222 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/B, priority=12, startTime=1732504480761; duration=0sec 2024-11-25T03:14:41,222 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:41,222 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:B 2024-11-25T03:14:41,222 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:14:41,230 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/20353384e0df45c08435e2683e37e3f6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/20353384e0df45c08435e2683e37e3f6 2024-11-25T03:14:41,237 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:14:41,237 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/C is initiating minor compaction (all files) 2024-11-25T03:14:41,237 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/C in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:41,237 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/09419ffd50a44c1baf8f3d800d67dab0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a4c3b6ccc10d43b6a4123e4de5cc4a74, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b072e729f4054a87b8af81064a00d29f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/d1e3530627be4f8bb4a68cc003397d30] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=48.6 K 2024-11-25T03:14:41,238 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 09419ffd50a44c1baf8f3d800d67dab0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504475716 2024-11-25T03:14:41,239 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c3b6ccc10d43b6a4123e4de5cc4a74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732504476944 2024-11-25T03:14:41,240 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b072e729f4054a87b8af81064a00d29f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732504477579 2024-11-25T03:14:41,241 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e3530627be4f8bb4a68cc003397d30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732504478334 2024-11-25T03:14:41,245 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/A of 73fa306731753442713ff2a6a1c4403b into 20353384e0df45c08435e2683e37e3f6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:41,245 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:41,245 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/A, priority=12, startTime=1732504480761; duration=0sec 2024-11-25T03:14:41,245 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:41,245 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:A 2024-11-25T03:14:41,255 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:41,256 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/352b2c97664741449d078f25146b50e0 is 50, key is test_row_0/C:col10/1732504478350/Put/seqid=0 2024-11-25T03:14:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504541249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504541251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504541254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504541255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742049_1225 (size=13017) 2024-11-25T03:14:41,277 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/515ea4f458544d3fad756af054cc1f2c 2024-11-25T03:14:41,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/856288acfded41f58a354c21315476ad is 50, key is test_row_0/B:col10/1732504479060/Put/seqid=0 2024-11-25T03:14:41,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742050_1226 (size=12301) 2024-11-25T03:14:41,316 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/856288acfded41f58a354c21315476ad 2024-11-25T03:14:41,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/453a50e200e74e34a0b7d2362b45dbe1 is 50, key is test_row_0/C:col10/1732504479060/Put/seqid=0 2024-11-25T03:14:41,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742051_1227 (size=12301) 2024-11-25T03:14:41,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504541360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504541370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504541371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504541371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504541570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504541590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504541590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504541590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:41,686 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/352b2c97664741449d078f25146b50e0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352b2c97664741449d078f25146b50e0 2024-11-25T03:14:41,694 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/C of 73fa306731753442713ff2a6a1c4403b into 352b2c97664741449d078f25146b50e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:41,694 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:41,694 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/C, priority=12, startTime=1732504480761; duration=0sec 2024-11-25T03:14:41,694 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:41,694 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:C 2024-11-25T03:14:41,731 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/453a50e200e74e34a0b7d2362b45dbe1 2024-11-25T03:14:41,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/515ea4f458544d3fad756af054cc1f2c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/515ea4f458544d3fad756af054cc1f2c 2024-11-25T03:14:41,744 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/515ea4f458544d3fad756af054cc1f2c, entries=150, sequenceid=368, filesize=30.5 K 2024-11-25T03:14:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/856288acfded41f58a354c21315476ad as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/856288acfded41f58a354c21315476ad 2024-11-25T03:14:41,751 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/856288acfded41f58a354c21315476ad, entries=150, sequenceid=368, filesize=12.0 K 2024-11-25T03:14:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/453a50e200e74e34a0b7d2362b45dbe1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/453a50e200e74e34a0b7d2362b45dbe1 2024-11-25T03:14:41,757 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/453a50e200e74e34a0b7d2362b45dbe1, entries=150, sequenceid=368, filesize=12.0 K 2024-11-25T03:14:41,758 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 73fa306731753442713ff2a6a1c4403b in 930ms, sequenceid=368, compaction requested=false 2024-11-25T03:14:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-25T03:14:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-25T03:14:41,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-25T03:14:41,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7850 sec 2024-11-25T03:14:41,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.7910 sec 2024-11-25T03:14:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:41,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=145.36 KB heapSize=415.77 KB 2024-11-25T03:14:41,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504481244, max timestamp=1732504481921Num uniques 1118; 2024-11-25T03:14:41,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-25T03:14:41,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504481244, max timestamp=1732504481921Num uniques 1018; 2024-11-25T03:14:41,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732504481244, max timestamp=1732504481921Num uniques 1018; 2024-11-25T03:14:41,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-25T03:14:41,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-25T03:14:41,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:41,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:41,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:41,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:41,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e992630d7a6d4480a6476e91a46b8b16_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504481948/Put/seqid=0 2024-11-25T03:14:41,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742052_1228 (size=14994) 2024-11-25T03:14:41,991 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:41,995 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e992630d7a6d4480a6476e91a46b8b16_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e992630d7a6d4480a6476e91a46b8b16_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:41,998 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/17bbf43e9cd94c44aae6c8490afb837e, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:41,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/17bbf43e9cd94c44aae6c8490afb837e is 175, key is test_row_0/A:col10/1732504481948/Put/seqid=0 2024-11-25T03:14:42,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504481986, max timestamp=1732504481997Num uniques 859; 2024-11-25T03:14:42,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:14:42,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504481986, max timestamp=1732504481997Num uniques 859; 2024-11-25T03:14:42,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504481986, max timestamp=1732504481997Num uniques 859; 2024-11-25T03:14:42,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:14:42,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:14:42,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742053_1229 (size=39949) 2024-11-25T03:14:42,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732504482022, max timestamp=1732504482039Num uniques 1068; 2024-11-25T03:14:42,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732504482022, max timestamp=1732504482039Num uniques 1068; 2024-11-25T03:14:42,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-25T03:14:42,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-25T03:14:42,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732504482022, max timestamp=1732504482039Num uniques 1068; 2024-11-25T03:14:42,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-25T03:14:42,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504542068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732504482056, max timestamp=1732504482073Num uniques 1127; 2024-11-25T03:14:42,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732504482056, max timestamp=1732504482073Num uniques 1127; 2024-11-25T03:14:42,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-25T03:14:42,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-25T03:14:42,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732504482056, max timestamp=1732504482073Num uniques 1127; 2024-11-25T03:14:42,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-25T03:14:42,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732504482110, max timestamp=1732504482133Num uniques 1136; 2024-11-25T03:14:42,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-25T03:14:42,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732504482110, max timestamp=1732504482133Num uniques 1136; 2024-11-25T03:14:42,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-25T03:14:42,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732504482110, max timestamp=1732504482133Num uniques 1136; 2024-11-25T03:14:42,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-25T03:14:42,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732504482214, max timestamp=1732504482245Num uniques 1095; 2024-11-25T03:14:42,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-25T03:14:42,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732504482214, max timestamp=1732504482245Num uniques 1095; 2024-11-25T03:14:42,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-25T03:14:42,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732504482214, max timestamp=1732504482245Num uniques 1095; 2024-11-25T03:14:42,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-25T03:14:42,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504542287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504542298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504542294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504542296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504542400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,407 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/17bbf43e9cd94c44aae6c8490afb837e 2024-11-25T03:14:42,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504542410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504542410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504542410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/34ec5c264fd04626a574ed0b26d08f40 is 50, key is test_row_0/B:col10/1732504481948/Put/seqid=0 2024-11-25T03:14:42,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742054_1230 (size=12301) 2024-11-25T03:14:42,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/34ec5c264fd04626a574ed0b26d08f40 2024-11-25T03:14:42,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/352740a8a2d440b1966ceeb3c2ebe695 is 50, key is test_row_0/C:col10/1732504481948/Put/seqid=0 2024-11-25T03:14:42,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742055_1231 (size=12301) 2024-11-25T03:14:42,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54742 deadline: 1732504542590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,597 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 73fa306731753442713ff2a6a1c4403b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18372 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:14:42,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504542610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504542620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504542630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504542630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/352740a8a2d440b1966ceeb3c2ebe695 2024-11-25T03:14:42,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/17bbf43e9cd94c44aae6c8490afb837e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/17bbf43e9cd94c44aae6c8490afb837e 2024-11-25T03:14:42,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/17bbf43e9cd94c44aae6c8490afb837e, entries=200, sequenceid=410, filesize=39.0 K 2024-11-25T03:14:42,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/34ec5c264fd04626a574ed0b26d08f40 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/34ec5c264fd04626a574ed0b26d08f40 2024-11-25T03:14:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504542920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/34ec5c264fd04626a574ed0b26d08f40, entries=150, sequenceid=410, filesize=12.0 K 2024-11-25T03:14:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/352740a8a2d440b1966ceeb3c2ebe695 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352740a8a2d440b1966ceeb3c2ebe695 2024-11-25T03:14:42,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352740a8a2d440b1966ceeb3c2ebe695, entries=150, sequenceid=410, filesize=12.0 K 2024-11-25T03:14:42,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~37.22 KB/38112, heapSize ~93.59 KB/95840, currentSize=154.31 KB/158010 for 73fa306731753442713ff2a6a1c4403b in 989ms, sequenceid=410, compaction requested=true 2024-11-25T03:14:42,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:42,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:14:42,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:42,937 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:42,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:14:42,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:42,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 73fa306731753442713ff2a6a1c4403b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:14:42,937 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:42,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:42,938 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:42,938 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/A is initiating minor compaction (all files) 2024-11-25T03:14:42,938 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/A in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:42,938 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/20353384e0df45c08435e2683e37e3f6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/515ea4f458544d3fad756af054cc1f2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/17bbf43e9cd94c44aae6c8490afb837e] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=100.8 K 2024-11-25T03:14:42,938 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:42,939 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/20353384e0df45c08435e2683e37e3f6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/515ea4f458544d3fad756af054cc1f2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/17bbf43e9cd94c44aae6c8490afb837e] 2024-11-25T03:14:42,939 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:42,939 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/B is initiating minor compaction (all files) 2024-11-25T03:14:42,939 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/B in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:42,939 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/1225fa43e1f243f18c1062810a5e788e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/856288acfded41f58a354c21315476ad, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/34ec5c264fd04626a574ed0b26d08f40] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=36.7 K 2024-11-25T03:14:42,940 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20353384e0df45c08435e2683e37e3f6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732504478334 2024-11-25T03:14:42,941 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 1225fa43e1f243f18c1062810a5e788e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732504478334 2024-11-25T03:14:42,941 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 515ea4f458544d3fad756af054cc1f2c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732504479049 2024-11-25T03:14:42,941 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 856288acfded41f58a354c21315476ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732504479049 2024-11-25T03:14:42,942 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ec5c264fd04626a574ed0b26d08f40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732504481882 2024-11-25T03:14:42,942 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17bbf43e9cd94c44aae6c8490afb837e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732504481882 2024-11-25T03:14:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:42,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=174.43 KB heapSize=492.47 KB 2024-11-25T03:14:42,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504542949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504542949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:42,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:14:42,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:42,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:14:42,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:42,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:14:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,961 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:42,961 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:42,962 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/cf760035b07b49bc8e18162b3aa225d7 is 50, key is test_row_0/B:col10/1732504481948/Put/seqid=0 2024-11-25T03:14:42,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504542955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:42,985 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ec71b93df62c4ac9be8467587b6940cd_73fa306731753442713ff2a6a1c4403b store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:42,986 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ec71b93df62c4ac9be8467587b6940cd_73fa306731753442713ff2a6a1c4403b, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:42,987 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ec71b93df62c4ac9be8467587b6940cd_73fa306731753442713ff2a6a1c4403b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:42,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251e08b932fbc04dfcbee976e88d4c9aa2_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504482283/Put/seqid=0 2024-11-25T03:14:42,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742056_1232 (size=13119) 2024-11-25T03:14:43,008 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/cf760035b07b49bc8e18162b3aa225d7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/cf760035b07b49bc8e18162b3aa225d7 2024-11-25T03:14:43,014 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/B of 73fa306731753442713ff2a6a1c4403b into cf760035b07b49bc8e18162b3aa225d7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:43,014 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:43,014 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/B, priority=13, startTime=1732504482937; duration=0sec 2024-11-25T03:14:43,014 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:14:43,015 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:B 2024-11-25T03:14:43,015 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:14:43,016 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:14:43,016 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 73fa306731753442713ff2a6a1c4403b/C is initiating minor compaction (all files) 2024-11-25T03:14:43,016 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 73fa306731753442713ff2a6a1c4403b/C in TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,016 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352b2c97664741449d078f25146b50e0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/453a50e200e74e34a0b7d2362b45dbe1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352740a8a2d440b1966ceeb3c2ebe695] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp, totalSize=36.7 K 2024-11-25T03:14:43,016 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 352b2c97664741449d078f25146b50e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732504478334 2024-11-25T03:14:43,017 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 453a50e200e74e34a0b7d2362b45dbe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732504479049 2024-11-25T03:14:43,017 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 352740a8a2d440b1966ceeb3c2ebe695, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732504481882 2024-11-25T03:14:43,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742058_1234 (size=17534) 2024-11-25T03:14:43,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742057_1233 (size=4469) 2024-11-25T03:14:43,041 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,046 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#C#compaction#195 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:43,047 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/73e8017a86a040f58858af83ae2b7a18 is 50, key is test_row_0/C:col10/1732504481948/Put/seqid=0 2024-11-25T03:14:43,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251e08b932fbc04dfcbee976e88d4c9aa2_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251e08b932fbc04dfcbee976e88d4c9aa2_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:43,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/a4adca0d717a487f9bdfab2ce8a0ed3d, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:43,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/a4adca0d717a487f9bdfab2ce8a0ed3d is 175, key is test_row_0/A:col10/1732504482283/Put/seqid=0 2024-11-25T03:14:43,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504543068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504543068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504543070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742059_1235 (size=13119) 2024-11-25T03:14:43,090 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/73e8017a86a040f58858af83ae2b7a18 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/73e8017a86a040f58858af83ae2b7a18 2024-11-25T03:14:43,097 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/C of 73fa306731753442713ff2a6a1c4403b into 73e8017a86a040f58858af83ae2b7a18(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:43,097 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:43,097 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/C, priority=13, startTime=1732504482937; duration=0sec 2024-11-25T03:14:43,097 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:43,097 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:C 2024-11-25T03:14:43,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742060_1236 (size=48639) 2024-11-25T03:14:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:14:43,141 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:43,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-25T03:14:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:14:43,143 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:43,144 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:43,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:43,186 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:14:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:14:43,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504543280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504543280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504543281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T03:14:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54756 deadline: 1732504543441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,445 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 73fa306731753442713ff2a6a1c4403b#A#compaction#193 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:14:43,446 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c6fb573c0ec843cf8cd85ed6b145bb41 is 175, key is test_row_0/A:col10/1732504481948/Put/seqid=0 2024-11-25T03:14:43,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T03:14:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:43,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742061_1237 (size=32073) 2024-11-25T03:14:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:14:43,472 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/c6fb573c0ec843cf8cd85ed6b145bb41 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c6fb573c0ec843cf8cd85ed6b145bb41 2024-11-25T03:14:43,481 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73fa306731753442713ff2a6a1c4403b/A of 73fa306731753442713ff2a6a1c4403b into c6fb573c0ec843cf8cd85ed6b145bb41(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:14:43,482 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:43,482 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., storeName=73fa306731753442713ff2a6a1c4403b/A, priority=13, startTime=1732504482937; duration=0sec 2024-11-25T03:14:43,482 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:14:43,482 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 73fa306731753442713ff2a6a1c4403b:A 2024-11-25T03:14:43,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=520, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/a4adca0d717a487f9bdfab2ce8a0ed3d 2024-11-25T03:14:43,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/bbbaec0c11f940398ffbe7a24ef4b377 is 50, key is test_row_0/B:col10/1732504482283/Put/seqid=0 2024-11-25T03:14:43,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742062_1238 (size=12301) 2024-11-25T03:14:43,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/bbbaec0c11f940398ffbe7a24ef4b377 2024-11-25T03:14:43,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54690 deadline: 1732504543590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54726 deadline: 1732504543590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54766 deadline: 1732504543590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:14:43,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/a084dae2dec24831a38c97b327eb3967 is 50, key is test_row_0/C:col10/1732504482283/Put/seqid=0 2024-11-25T03:14:43,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T03:14:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. as already flushing 2024-11-25T03:14:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:43,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742063_1239 (size=12301) 2024-11-25T03:14:43,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/a084dae2dec24831a38c97b327eb3967 2024-11-25T03:14:43,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/a4adca0d717a487f9bdfab2ce8a0ed3d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/a4adca0d717a487f9bdfab2ce8a0ed3d 2024-11-25T03:14:43,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/a4adca0d717a487f9bdfab2ce8a0ed3d, entries=250, sequenceid=520, filesize=47.5 K 2024-11-25T03:14:43,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/bbbaec0c11f940398ffbe7a24ef4b377 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/bbbaec0c11f940398ffbe7a24ef4b377 2024-11-25T03:14:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/bbbaec0c11f940398ffbe7a24ef4b377, entries=150, sequenceid=520, filesize=12.0 K 2024-11-25T03:14:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/a084dae2dec24831a38c97b327eb3967 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a084dae2dec24831a38c97b327eb3967 2024-11-25T03:14:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a084dae2dec24831a38c97b327eb3967, entries=150, sequenceid=520, filesize=12.0 K 2024-11-25T03:14:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~512.34 KB/524640, currentSize=0 B/0 for 73fa306731753442713ff2a6a1c4403b in 755ms, sequenceid=520, compaction requested=false 2024-11-25T03:14:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T03:14:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-25T03:14:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-25T03:14:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-25T03:14:43,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 619 msec 2024-11-25T03:14:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 625 msec 2024-11-25T03:14:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:14:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,771 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-25T03:14:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:14:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,774 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:43,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] io.ByteBuffAllocator(384): Pool already reached its max capacity : 186 and no free buffers now. Consider increasing the value for 'hbase.server.allocator.max.buffer.count' ? 2024-11-25T03:14:43,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] io.ByteBuffAllocator(384): Pool already reached its max capacity : 186 and no free buffers now. Consider increasing the value for 'hbase.server.allocator.max.buffer.count' ? 2024-11-25T03:14:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:14:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T03:14:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 73fa306731753442713ff2a6a1c4403b: 2024-11-25T03:14:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-25T03:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-25T03:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T03:14:43,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-25T03:14:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 163 msec 2024-11-25T03:14:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,082 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-11-25T03:14:44,082 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,082 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,082 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,086 INFO [Thread-683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-683. 2024-11-25T03:14:44,086 DEBUG [Thread-683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,086 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,086 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:14:44,091 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,101 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-25T03:14:44,101 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,101 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,102 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,103 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-25T03:14:44,103 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,103 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,103 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,104 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-25T03:14:44,104 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,104 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,104 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,111 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-11-25T03:14:44,111 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,111 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,111 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:44,200 INFO [Thread-689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-689. 2024-11-25T03:14:44,200 DEBUG [Thread-689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,200 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,200 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:44,453 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-25T03:14:44,453 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:44,453 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,453 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:44,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:52,632 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-25T03:14:52,632 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:14:52,632 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:52,632 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:52,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2348 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2275 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:14:52,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1016 2024-11-25T03:14:52,633 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3046 rows 2024-11-25T03:14:52,633 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1017 2024-11-25T03:14:52,633 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3049 rows 2024-11-25T03:14:52,633 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:14:52,633 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:14:52,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:52,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:52,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:14:52,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T03:14:52,637 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504492636"}]},"ts":"1732504492636"} 2024-11-25T03:14:52,638 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:14:52,638 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:14:52,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:14:52,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, UNASSIGN}] 2024-11-25T03:14:52,641 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, UNASSIGN 2024-11-25T03:14:52,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:52,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, UNASSIGN because future has completed 2024-11-25T03:14:52,645 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:14:52,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T03:14:52,797 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:52,797 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 73fa306731753442713ff2a6a1c4403b, disabling compactions & flushes 2024-11-25T03:14:52,798 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. after waiting 0 ms 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:52,798 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 73fa306731753442713ff2a6a1c4403b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=A 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=B 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 73fa306731753442713ff2a6a1c4403b, store=C 2024-11-25T03:14:52,798 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:52,830 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254f3f3777430a46c3b83e571956ccd6ef_73fa306731753442713ff2a6a1c4403b is 50, key is test_row_0/A:col10/1732504492630/Put/seqid=0 2024-11-25T03:14:52,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742064_1240 (size=9914) 2024-11-25T03:14:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T03:14:53,240 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:53,245 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254f3f3777430a46c3b83e571956ccd6ef_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f3f3777430a46c3b83e571956ccd6ef_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:53,246 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/3b288374aead4d4a8f2550af8e7d4c25, store: [table=TestAcidGuarantees family=A region=73fa306731753442713ff2a6a1c4403b] 2024-11-25T03:14:53,246 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/3b288374aead4d4a8f2550af8e7d4c25 is 175, key is test_row_0/A:col10/1732504492630/Put/seqid=0 2024-11-25T03:14:53,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742065_1241 (size=22561) 2024-11-25T03:14:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T03:14:53,650 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=533, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/3b288374aead4d4a8f2550af8e7d4c25 2024-11-25T03:14:53,657 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/b608fb0a8f4f4e93a7e09242c1f507cf is 50, key is test_row_0/B:col10/1732504492630/Put/seqid=0 2024-11-25T03:14:53,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742066_1242 (size=9857) 2024-11-25T03:14:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T03:14:54,061 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/b608fb0a8f4f4e93a7e09242c1f507cf 2024-11-25T03:14:54,068 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/3c470c5f0ea04943affb95dc49fe0b07 is 50, key is test_row_0/C:col10/1732504492630/Put/seqid=0 2024-11-25T03:14:54,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742067_1243 (size=9857) 2024-11-25T03:14:54,472 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/3c470c5f0ea04943affb95dc49fe0b07 2024-11-25T03:14:54,477 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/A/3b288374aead4d4a8f2550af8e7d4c25 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/3b288374aead4d4a8f2550af8e7d4c25 2024-11-25T03:14:54,481 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/3b288374aead4d4a8f2550af8e7d4c25, entries=100, sequenceid=533, filesize=22.0 K 2024-11-25T03:14:54,482 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/B/b608fb0a8f4f4e93a7e09242c1f507cf as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b608fb0a8f4f4e93a7e09242c1f507cf 2024-11-25T03:14:54,485 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b608fb0a8f4f4e93a7e09242c1f507cf, entries=100, sequenceid=533, filesize=9.6 K 2024-11-25T03:14:54,486 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/.tmp/C/3c470c5f0ea04943affb95dc49fe0b07 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c470c5f0ea04943affb95dc49fe0b07 2024-11-25T03:14:54,489 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c470c5f0ea04943affb95dc49fe0b07, entries=100, sequenceid=533, filesize=9.6 K 2024-11-25T03:14:54,489 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 73fa306731753442713ff2a6a1c4403b in 1691ms, sequenceid=533, compaction requested=true 2024-11-25T03:14:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/e302c7e28462441fb4dd859f1e4b7bc1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c80f38d4fb0a4e458ab65f063b5d15b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9992bb570e94316b1a3d77ca262be3f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9a9b6d6b2564ef288731a36d81a2172, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/6b4e6831e1f5424ca6e7e2268bb25760, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/572344752f72463e8799d64aa0734f1b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/07c941c1e91943deb826a37c0eef4457, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/34e0d3a884cb4fa49caecb6f0abbcdac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/0bcc33e8ed6f41889195a45a326f211c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/339d8693797a4cf7ae5a04bfb004bfd9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c0563dae45eb4362bac448c0873c1a0e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/d7497abf4e66476ca3a1ca75febb41fe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/612b7c5cae294059b01490aeda61e8c3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/ae284665d3d8451cb26da5045b373da2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/8433527e1f0f4e9fac93a0a459c493fb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/51ca08f67f2b448aa25d078ecf5ca548, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/33984fbe41d544dd988d0a3f9945e3ff, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/aeb65263cc5744238d9db0f8bfa4df6f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/780158e69516422fb02be48fc2f41e13, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/9e35f1c7f1f04f8682111624c680f3b3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/79cc678ee1e74db5b1db1915a6edd310, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/20353384e0df45c08435e2683e37e3f6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/515ea4f458544d3fad756af054cc1f2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/17bbf43e9cd94c44aae6c8490afb837e] to archive 2024-11-25T03:14:54,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:14:54,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/e302c7e28462441fb4dd859f1e4b7bc1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/e302c7e28462441fb4dd859f1e4b7bc1 2024-11-25T03:14:54,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c80f38d4fb0a4e458ab65f063b5d15b3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c80f38d4fb0a4e458ab65f063b5d15b3 2024-11-25T03:14:54,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9992bb570e94316b1a3d77ca262be3f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9992bb570e94316b1a3d77ca262be3f 2024-11-25T03:14:54,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9a9b6d6b2564ef288731a36d81a2172 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/b9a9b6d6b2564ef288731a36d81a2172 2024-11-25T03:14:54,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/6b4e6831e1f5424ca6e7e2268bb25760 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/6b4e6831e1f5424ca6e7e2268bb25760 2024-11-25T03:14:54,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/572344752f72463e8799d64aa0734f1b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/572344752f72463e8799d64aa0734f1b 2024-11-25T03:14:54,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/07c941c1e91943deb826a37c0eef4457 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/07c941c1e91943deb826a37c0eef4457 2024-11-25T03:14:54,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/34e0d3a884cb4fa49caecb6f0abbcdac to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/34e0d3a884cb4fa49caecb6f0abbcdac 2024-11-25T03:14:54,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/0bcc33e8ed6f41889195a45a326f211c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/0bcc33e8ed6f41889195a45a326f211c 2024-11-25T03:14:54,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/339d8693797a4cf7ae5a04bfb004bfd9 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/339d8693797a4cf7ae5a04bfb004bfd9 2024-11-25T03:14:54,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c0563dae45eb4362bac448c0873c1a0e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c0563dae45eb4362bac448c0873c1a0e 2024-11-25T03:14:54,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/d7497abf4e66476ca3a1ca75febb41fe to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/d7497abf4e66476ca3a1ca75febb41fe 2024-11-25T03:14:54,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/612b7c5cae294059b01490aeda61e8c3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/612b7c5cae294059b01490aeda61e8c3 2024-11-25T03:14:54,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/ae284665d3d8451cb26da5045b373da2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/ae284665d3d8451cb26da5045b373da2 2024-11-25T03:14:54,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/8433527e1f0f4e9fac93a0a459c493fb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/8433527e1f0f4e9fac93a0a459c493fb 2024-11-25T03:14:54,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/51ca08f67f2b448aa25d078ecf5ca548 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/51ca08f67f2b448aa25d078ecf5ca548 2024-11-25T03:14:54,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/33984fbe41d544dd988d0a3f9945e3ff to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/33984fbe41d544dd988d0a3f9945e3ff 2024-11-25T03:14:54,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/aeb65263cc5744238d9db0f8bfa4df6f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/aeb65263cc5744238d9db0f8bfa4df6f 2024-11-25T03:14:54,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/780158e69516422fb02be48fc2f41e13 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/780158e69516422fb02be48fc2f41e13 2024-11-25T03:14:54,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/9e35f1c7f1f04f8682111624c680f3b3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/9e35f1c7f1f04f8682111624c680f3b3 2024-11-25T03:14:54,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/79cc678ee1e74db5b1db1915a6edd310 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/79cc678ee1e74db5b1db1915a6edd310 2024-11-25T03:14:54,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/20353384e0df45c08435e2683e37e3f6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/20353384e0df45c08435e2683e37e3f6 2024-11-25T03:14:54,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/515ea4f458544d3fad756af054cc1f2c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/515ea4f458544d3fad756af054cc1f2c 2024-11-25T03:14:54,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/17bbf43e9cd94c44aae6c8490afb837e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/17bbf43e9cd94c44aae6c8490afb837e 2024-11-25T03:14:54,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/d85b101c8296496d911a1ae676a8dd09, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/930df25eac904f50a469a2f3f5f91706, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef25de7fcee840bdb1fcc69da6688d86, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/da003b456d6446ec999ff47f7aa94424, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5f901af069a74fd8811ff8f0d5ff2f44, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/339c03e1f3fe4ba1be1051b1fad01480, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/a3d258d5bdc745b3961e2096cb8f594d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b080aee9db334155a7a31242e43e7fcc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f991ecfcd421432b91a4ce484624445d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef2e514affe74c76b66126a223ac375d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f00f70e7305b4b8eaf8a36abaa2502af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/eb561894d6bb4171802d78c446969f31, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5edc2df839654612b48b9dab5d422f73, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/2370006bd0264c5c937e8e353afe72ed, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/4f4f850e77b24ad6a1134477ff7b7c84, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/236c1a786bfd40bd8baa5c3be4cab92c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/be9fd7d01bdb426aa76ff6637cfe7105, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/95ab79eefcea4ec880950df14bb1659b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/c046f31c936d42f9b2b9e8128dafffd7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/1225fa43e1f243f18c1062810a5e788e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5e90d716ea7640bbb5f960e4cfd0efd4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/856288acfded41f58a354c21315476ad, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/34ec5c264fd04626a574ed0b26d08f40] to archive 2024-11-25T03:14:54,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:14:54,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/d85b101c8296496d911a1ae676a8dd09 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/d85b101c8296496d911a1ae676a8dd09 2024-11-25T03:14:54,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/930df25eac904f50a469a2f3f5f91706 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/930df25eac904f50a469a2f3f5f91706 2024-11-25T03:14:54,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef25de7fcee840bdb1fcc69da6688d86 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef25de7fcee840bdb1fcc69da6688d86 2024-11-25T03:14:54,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/da003b456d6446ec999ff47f7aa94424 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/da003b456d6446ec999ff47f7aa94424 2024-11-25T03:14:54,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/fcffd4d98fd14d6da52ac4c5ea1b4fc1 2024-11-25T03:14:54,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5f901af069a74fd8811ff8f0d5ff2f44 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5f901af069a74fd8811ff8f0d5ff2f44 2024-11-25T03:14:54,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/339c03e1f3fe4ba1be1051b1fad01480 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/339c03e1f3fe4ba1be1051b1fad01480 2024-11-25T03:14:54,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/a3d258d5bdc745b3961e2096cb8f594d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/a3d258d5bdc745b3961e2096cb8f594d 2024-11-25T03:14:54,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b080aee9db334155a7a31242e43e7fcc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b080aee9db334155a7a31242e43e7fcc 2024-11-25T03:14:54,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f991ecfcd421432b91a4ce484624445d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f991ecfcd421432b91a4ce484624445d 2024-11-25T03:14:54,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef2e514affe74c76b66126a223ac375d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/ef2e514affe74c76b66126a223ac375d 2024-11-25T03:14:54,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f00f70e7305b4b8eaf8a36abaa2502af to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/f00f70e7305b4b8eaf8a36abaa2502af 2024-11-25T03:14:54,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/eb561894d6bb4171802d78c446969f31 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/eb561894d6bb4171802d78c446969f31 2024-11-25T03:14:54,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5edc2df839654612b48b9dab5d422f73 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5edc2df839654612b48b9dab5d422f73 2024-11-25T03:14:54,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/2370006bd0264c5c937e8e353afe72ed to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/2370006bd0264c5c937e8e353afe72ed 2024-11-25T03:14:54,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/4f4f850e77b24ad6a1134477ff7b7c84 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/4f4f850e77b24ad6a1134477ff7b7c84 2024-11-25T03:14:54,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/236c1a786bfd40bd8baa5c3be4cab92c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/236c1a786bfd40bd8baa5c3be4cab92c 2024-11-25T03:14:54,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/be9fd7d01bdb426aa76ff6637cfe7105 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/be9fd7d01bdb426aa76ff6637cfe7105 2024-11-25T03:14:54,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/95ab79eefcea4ec880950df14bb1659b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/95ab79eefcea4ec880950df14bb1659b 2024-11-25T03:14:54,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/c046f31c936d42f9b2b9e8128dafffd7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/c046f31c936d42f9b2b9e8128dafffd7 2024-11-25T03:14:54,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/1225fa43e1f243f18c1062810a5e788e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/1225fa43e1f243f18c1062810a5e788e 2024-11-25T03:14:54,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5e90d716ea7640bbb5f960e4cfd0efd4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/5e90d716ea7640bbb5f960e4cfd0efd4 2024-11-25T03:14:54,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/856288acfded41f58a354c21315476ad to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/856288acfded41f58a354c21315476ad 2024-11-25T03:14:54,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/34ec5c264fd04626a574ed0b26d08f40 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/34ec5c264fd04626a574ed0b26d08f40 2024-11-25T03:14:54,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/33ab486886df4e209f8148bc40c96aac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/8a646ef51a844b9f8b9af821c595cadd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/54401ca2d6bf40c087edc40caf03547b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/4818c87fd0f94d30a1daf4f02ce84db3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/5dcf993de6ef4838823219bb40eadfde, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/05b57687e7d946289d7ee013287e0824, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b3550586975b45388da62c98ed31e436, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c1bafd690d245ba86e78ce8d7144bdc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/ca54304f350747d790f5e8ae58b98150, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/da527e11f30b4fa3ba3e61635ef622e6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/802ae0e31c87401cab86aa8f50d5804b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/36fe1e2bee9141a5ba0b0b511cd80bf0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/cb26280cd1594a9fa994a6a24ec81afe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/eafe3618084b4d43a17fa49d9eed0267, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/e68760663a734edea3722d5825563aad, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/2d4666763d7242a29e5316668a3ce348, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/09419ffd50a44c1baf8f3d800d67dab0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/70bab88d80c2460ba61c78b33a086701, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a4c3b6ccc10d43b6a4123e4de5cc4a74, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b072e729f4054a87b8af81064a00d29f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352b2c97664741449d078f25146b50e0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/d1e3530627be4f8bb4a68cc003397d30, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/453a50e200e74e34a0b7d2362b45dbe1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352740a8a2d440b1966ceeb3c2ebe695] to archive 2024-11-25T03:14:54,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:14:54,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/33ab486886df4e209f8148bc40c96aac to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/33ab486886df4e209f8148bc40c96aac 2024-11-25T03:14:54,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/8a646ef51a844b9f8b9af821c595cadd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/8a646ef51a844b9f8b9af821c595cadd 2024-11-25T03:14:54,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/54401ca2d6bf40c087edc40caf03547b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/54401ca2d6bf40c087edc40caf03547b 2024-11-25T03:14:54,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/4818c87fd0f94d30a1daf4f02ce84db3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/4818c87fd0f94d30a1daf4f02ce84db3 2024-11-25T03:14:54,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/5dcf993de6ef4838823219bb40eadfde to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/5dcf993de6ef4838823219bb40eadfde 2024-11-25T03:14:54,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/05b57687e7d946289d7ee013287e0824 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/05b57687e7d946289d7ee013287e0824 2024-11-25T03:14:54,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b3550586975b45388da62c98ed31e436 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b3550586975b45388da62c98ed31e436 2024-11-25T03:14:54,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c1bafd690d245ba86e78ce8d7144bdc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c1bafd690d245ba86e78ce8d7144bdc 2024-11-25T03:14:54,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/ca54304f350747d790f5e8ae58b98150 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/ca54304f350747d790f5e8ae58b98150 2024-11-25T03:14:54,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/da527e11f30b4fa3ba3e61635ef622e6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/da527e11f30b4fa3ba3e61635ef622e6 2024-11-25T03:14:54,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/802ae0e31c87401cab86aa8f50d5804b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/802ae0e31c87401cab86aa8f50d5804b 2024-11-25T03:14:54,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/36fe1e2bee9141a5ba0b0b511cd80bf0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/36fe1e2bee9141a5ba0b0b511cd80bf0 2024-11-25T03:14:54,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/cb26280cd1594a9fa994a6a24ec81afe to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/cb26280cd1594a9fa994a6a24ec81afe 2024-11-25T03:14:54,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/eafe3618084b4d43a17fa49d9eed0267 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/eafe3618084b4d43a17fa49d9eed0267 2024-11-25T03:14:54,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/e68760663a734edea3722d5825563aad to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/e68760663a734edea3722d5825563aad 2024-11-25T03:14:54,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/2d4666763d7242a29e5316668a3ce348 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/2d4666763d7242a29e5316668a3ce348 2024-11-25T03:14:54,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/09419ffd50a44c1baf8f3d800d67dab0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/09419ffd50a44c1baf8f3d800d67dab0 2024-11-25T03:14:54,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/70bab88d80c2460ba61c78b33a086701 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/70bab88d80c2460ba61c78b33a086701 2024-11-25T03:14:54,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a4c3b6ccc10d43b6a4123e4de5cc4a74 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a4c3b6ccc10d43b6a4123e4de5cc4a74 2024-11-25T03:14:54,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b072e729f4054a87b8af81064a00d29f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/b072e729f4054a87b8af81064a00d29f 2024-11-25T03:14:54,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352b2c97664741449d078f25146b50e0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352b2c97664741449d078f25146b50e0 2024-11-25T03:14:54,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/d1e3530627be4f8bb4a68cc003397d30 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/d1e3530627be4f8bb4a68cc003397d30 2024-11-25T03:14:54,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/453a50e200e74e34a0b7d2362b45dbe1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/453a50e200e74e34a0b7d2362b45dbe1 2024-11-25T03:14:54,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352740a8a2d440b1966ceeb3c2ebe695 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/352740a8a2d440b1966ceeb3c2ebe695 2024-11-25T03:14:54,572 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/recovered.edits/536.seqid, newMaxSeqId=536, maxSeqId=4 2024-11-25T03:14:54,573 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. 2024-11-25T03:14:54,573 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 73fa306731753442713ff2a6a1c4403b: Waiting for close lock at 1732504492797Running coprocessor pre-close hooks at 1732504492797Disabling compacts and flushes for region at 1732504492797Disabling writes for close at 1732504492798 (+1 ms)Obtaining lock to block concurrent updates at 1732504492798Preparing flush snapshotting stores in 73fa306731753442713ff2a6a1c4403b at 1732504492798Finished memstore snapshotting TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732504492799 (+1 ms)Flushing stores of TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b. at 1732504492799Flushing 73fa306731753442713ff2a6a1c4403b/A: creating writer at 1732504492799Flushing mob file 73fa306731753442713ff2a6a1c4403b/A: appending metadata at 1732504492829 (+30 ms)Flushing mob file 73fa306731753442713ff2a6a1c4403b/A: closing flushed file at 1732504492829Flushing 73fa306731753442713ff2a6a1c4403b/A: appending metadata at 1732504493246 (+417 ms)Flushing 73fa306731753442713ff2a6a1c4403b/A: closing flushed file at 1732504493246Flushing 73fa306731753442713ff2a6a1c4403b/B: creating writer at 1732504493654 (+408 ms)Flushing 73fa306731753442713ff2a6a1c4403b/B: appending metadata at 1732504493657 (+3 ms)Flushing 73fa306731753442713ff2a6a1c4403b/B: closing flushed file at 1732504493657Flushing 73fa306731753442713ff2a6a1c4403b/C: creating writer at 1732504494065 (+408 ms)Flushing 73fa306731753442713ff2a6a1c4403b/C: appending metadata at 1732504494068 (+3 ms)Flushing 73fa306731753442713ff2a6a1c4403b/C: closing flushed file at 1732504494068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195f87c1: reopening flushed file at 1732504494476 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31a92b95: reopening flushed file at 1732504494481 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63fed7d4: reopening flushed file at 1732504494485 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 73fa306731753442713ff2a6a1c4403b in 1691ms, sequenceid=533, compaction requested=true at 1732504494489 (+4 ms)Writing region close event to WAL at 1732504494569 (+80 ms)Running coprocessor post-close hooks at 1732504494573 (+4 ms)Closed at 1732504494573 2024-11-25T03:14:54,575 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,576 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=73fa306731753442713ff2a6a1c4403b, regionState=CLOSED 2024-11-25T03:14:54,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:14:54,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-25T03:14:54,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 73fa306731753442713ff2a6a1c4403b, server=38a5d2077268,43357,1732504425108 in 1.9320 sec 2024-11-25T03:14:54,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-25T03:14:54,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=73fa306731753442713ff2a6a1c4403b, UNASSIGN in 1.9390 sec 2024-11-25T03:14:54,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-25T03:14:54,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9420 sec 2024-11-25T03:14:54,583 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504494583"}]},"ts":"1732504494583"} 2024-11-25T03:14:54,584 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:14:54,584 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:14:54,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9510 sec 2024-11-25T03:14:54,756 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T03:14:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T03:14:54,790 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:54,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:54,792 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T03:14:54,793 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:54,796 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,798 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/recovered.edits] 2024-11-25T03:14:54,800 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/3b288374aead4d4a8f2550af8e7d4c25 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/3b288374aead4d4a8f2550af8e7d4c25 2024-11-25T03:14:54,801 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/a4adca0d717a487f9bdfab2ce8a0ed3d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/a4adca0d717a487f9bdfab2ce8a0ed3d 2024-11-25T03:14:54,802 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c6fb573c0ec843cf8cd85ed6b145bb41 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/A/c6fb573c0ec843cf8cd85ed6b145bb41 2024-11-25T03:14:54,804 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b608fb0a8f4f4e93a7e09242c1f507cf to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/b608fb0a8f4f4e93a7e09242c1f507cf 2024-11-25T03:14:54,805 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/bbbaec0c11f940398ffbe7a24ef4b377 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/bbbaec0c11f940398ffbe7a24ef4b377 2024-11-25T03:14:54,806 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/cf760035b07b49bc8e18162b3aa225d7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/B/cf760035b07b49bc8e18162b3aa225d7 2024-11-25T03:14:54,808 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c470c5f0ea04943affb95dc49fe0b07 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/3c470c5f0ea04943affb95dc49fe0b07 2024-11-25T03:14:54,809 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/73e8017a86a040f58858af83ae2b7a18 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/73e8017a86a040f58858af83ae2b7a18 2024-11-25T03:14:54,810 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a084dae2dec24831a38c97b327eb3967 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/C/a084dae2dec24831a38c97b327eb3967 2024-11-25T03:14:54,812 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/recovered.edits/536.seqid to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b/recovered.edits/536.seqid 2024-11-25T03:14:54,812 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,813 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:14:54,813 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:14:54,814 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T03:14:54,817 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251d96316b1edf4664ba1cbc674fb1b076_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251d96316b1edf4664ba1cbc674fb1b076_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,818 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251e08b932fbc04dfcbee976e88d4c9aa2_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251e08b932fbc04dfcbee976e88d4c9aa2_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,819 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522c18540d26f4893a36676966856af6e_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522c18540d26f4893a36676966856af6e_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252482868054ca4056bfa8eb511670347c_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252482868054ca4056bfa8eb511670347c_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,822 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125256b4cfe9c204b42a3cabc7bc97a6521_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125256b4cfe9c204b42a3cabc7bc97a6521_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,823 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112543c6ba20653a40e184b12870b28d4a21_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112543c6ba20653a40e184b12870b28d4a21_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,824 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f3f3777430a46c3b83e571956ccd6ef_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f3f3777430a46c3b83e571956ccd6ef_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,825 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256174f2e15fdf4c27a5bebcb221faca48_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256174f2e15fdf4c27a5bebcb221faca48_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,826 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112564b337ac2bcd4353bdd1e1e4ee5e075e_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112564b337ac2bcd4353bdd1e1e4ee5e075e_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,827 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112568428c32cb314446a852b2b76e95935e_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112568428c32cb314446a852b2b76e95935e_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,828 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112581e164bb27c14fbaa67c97a26f316150_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112581e164bb27c14fbaa67c97a26f316150_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,829 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112592628547a835493288778e833e15dc7f_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112592628547a835493288778e833e15dc7f_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,830 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259b7b04f32c1d4dcba9a569a81bb2319d_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259b7b04f32c1d4dcba9a569a81bb2319d_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,831 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c0b63bd8b09b490d924409827d2df996_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c0b63bd8b09b490d924409827d2df996_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,832 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c78c993568ee4a1f9736f8887f0bb2c3_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c78c993568ee4a1f9736f8887f0bb2c3_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,833 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cc475db629f5454782fda330bb9a4138_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cc475db629f5454782fda330bb9a4138_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,834 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d1733fe082a640f7a7257414753851c2_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d1733fe082a640f7a7257414753851c2_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d7b92ef6bb1b4301bb398486bc91f975_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d7b92ef6bb1b4301bb398486bc91f975_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,836 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dcd0fd9e6bec4973881288c764bcb14b_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dcd0fd9e6bec4973881288c764bcb14b_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,837 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e992630d7a6d4480a6476e91a46b8b16_73fa306731753442713ff2a6a1c4403b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e992630d7a6d4480a6476e91a46b8b16_73fa306731753442713ff2a6a1c4403b 2024-11-25T03:14:54,838 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:14:54,840 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:54,843 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:14:54,845 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:14:54,846 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:54,846 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:14:54,846 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732504494846"}]},"ts":"9223372036854775807"} 2024-11-25T03:14:54,849 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:14:54,849 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 73fa306731753442713ff2a6a1c4403b, NAME => 'TestAcidGuarantees,,1732504460885.73fa306731753442713ff2a6a1c4403b.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:14:54,849 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:14:54,849 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732504494849"}]},"ts":"9223372036854775807"} 2024-11-25T03:14:54,851 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:14:54,852 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:54,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-25T03:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T03:14:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:14:54,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:54,909 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 243) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-34767401_22 at /127.0.0.1:35290 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=457 (was 341) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7082 (was 7586) 2024-11-25T03:14:54,918 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=457, ProcessCount=11, AvailableMemoryMB=7082 2024-11-25T03:14:54,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:14:54,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:14:54,921 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:14:54,921 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:54,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-25T03:14:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T03:14:54,922 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:14:54,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742068_1244 (size=1061) 2024-11-25T03:14:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T03:14:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T03:14:55,329 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:14:55,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742069_1245 (size=53) 2024-11-25T03:14:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T03:14:55,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:14:55,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 897f6457507f38ffcd25c9bdd6308219, disabling compactions & flushes 2024-11-25T03:14:55,735 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:55,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:55,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. after waiting 0 ms 2024-11-25T03:14:55,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:55,735 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:55,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 897f6457507f38ffcd25c9bdd6308219: Waiting for close lock at 1732504495735Disabling compacts and flushes for region at 1732504495735Disabling writes for close at 1732504495735Writing region close event to WAL at 1732504495735Closed at 1732504495735 2024-11-25T03:14:55,736 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:14:55,737 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732504495736"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732504495736"}]},"ts":"1732504495736"} 2024-11-25T03:14:55,738 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:14:55,739 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:14:55,739 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504495739"}]},"ts":"1732504495739"} 2024-11-25T03:14:55,740 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:14:55,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, ASSIGN}] 2024-11-25T03:14:55,741 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, ASSIGN 2024-11-25T03:14:55,741 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, ASSIGN; state=OFFLINE, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=false 2024-11-25T03:14:55,892 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=897f6457507f38ffcd25c9bdd6308219, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:55,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, ASSIGN because future has completed 2024-11-25T03:14:55,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:14:56,048 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:56,049 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:14:56,049 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,049 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:14:56,049 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,049 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,051 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,052 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:56,052 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 897f6457507f38ffcd25c9bdd6308219 columnFamilyName A 2024-11-25T03:14:56,052 DEBUG [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:56,052 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.HStore(327): Store=897f6457507f38ffcd25c9bdd6308219/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:56,052 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,053 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:56,054 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 897f6457507f38ffcd25c9bdd6308219 columnFamilyName B 2024-11-25T03:14:56,054 DEBUG [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:56,054 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.HStore(327): Store=897f6457507f38ffcd25c9bdd6308219/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:56,054 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,055 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:14:56,055 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 897f6457507f38ffcd25c9bdd6308219 columnFamilyName C 2024-11-25T03:14:56,055 DEBUG [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:14:56,056 INFO [StoreOpener-897f6457507f38ffcd25c9bdd6308219-1 {}] regionserver.HStore(327): Store=897f6457507f38ffcd25c9bdd6308219/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:14:56,056 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:56,056 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,056 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,057 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,057 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,057 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,058 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:14:56,059 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T03:14:56,060 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:14:56,061 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 897f6457507f38ffcd25c9bdd6308219; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61287383, jitterRate=-0.08674682676792145}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:14:56,061 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:56,062 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 897f6457507f38ffcd25c9bdd6308219: Running coprocessor pre-open hook at 1732504496049Writing region info on filesystem at 1732504496049Initializing all the Stores at 1732504496050 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504496050Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504496050Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504496050Cleaning up temporary data from old regions at 1732504496057 (+7 ms)Running coprocessor post-open hooks at 1732504496061 (+4 ms)Region opened successfully at 1732504496061 2024-11-25T03:14:56,062 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., pid=69, masterSystemTime=1732504496046 2024-11-25T03:14:56,064 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:56,064 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:56,064 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=897f6457507f38ffcd25c9bdd6308219, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:14:56,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:14:56,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-25T03:14:56,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 in 172 msec 2024-11-25T03:14:56,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-25T03:14:56,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, ASSIGN in 328 msec 2024-11-25T03:14:56,070 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:14:56,070 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504496070"}]},"ts":"1732504496070"} 2024-11-25T03:14:56,071 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:14:56,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:14:56,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-25T03:14:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T03:14:57,070 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:57,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,072 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a567773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,074 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62734d81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,075 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,078 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:14:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,44623,1732504424313 2024-11-25T03:14:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63371c57 2024-11-25T03:14:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:14:57,079 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:14:57,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,081 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf02d41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,082 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d796cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,084 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fe4796c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,085 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4dbccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,087 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d111041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c879e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,090 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5344223f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,091 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65db4733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,092 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44539cc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,094 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3eb54f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,095 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7057d688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,097 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6c15b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f670c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517bd56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,101 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e44681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,103 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357b0846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,104 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b85287e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3f3282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:14:57,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:14:57,107 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:14:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:14:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:14:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20aafee7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:14:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:14:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:14:57,108 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:14:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45251b5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:14:57,109 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,109 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,109 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,109 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,110 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,110 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-25T03:14:57,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,112 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:57,113 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,113 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:57,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:57,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:14:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,120 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,120 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,121 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,122 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:14:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:14:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:14:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:14:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:14:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:14:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:14:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:14:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:14:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:57,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/106700eb075d4a7c909afe92fdd0fed2 is 50, key is test_row_0/A:col10/1732504497128/Put/seqid=0 2024-11-25T03:14:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504557162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504557164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504557165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504497159, max timestamp=1732504497168Num uniques 859; 2024-11-25T03:14:57,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504557164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504497159, max timestamp=1732504497168Num uniques 859; 2024-11-25T03:14:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504497159, max timestamp=1732504497168Num uniques 859; 2024-11-25T03:14:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742070_1246 (size=12001) 2024-11-25T03:14:57,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/106700eb075d4a7c909afe92fdd0fed2 2024-11-25T03:14:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:14:57,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8f40fd47d4cb42768a5f6d3652efba83 is 50, key is test_row_0/B:col10/1732504497128/Put/seqid=0 2024-11-25T03:14:57,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742071_1247 (size=12001) 2024-11-25T03:14:57,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8f40fd47d4cb42768a5f6d3652efba83 2024-11-25T03:14:57,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T03:14:57,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/0821195674f1491da959f2cef31102d1 is 50, key is test_row_0/C:col10/1732504497128/Put/seqid=0 2024-11-25T03:14:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:14:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:57,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742072_1248 (size=12001) 2024-11-25T03:14:57,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504557280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504557280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504557281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504557281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504557281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504557390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T03:14:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:14:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:57,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:14:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504557490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504557490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504557490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504557490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T03:14:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:14:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:57,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:14:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504557599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/0821195674f1491da959f2cef31102d1 2024-11-25T03:14:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/106700eb075d4a7c909afe92fdd0fed2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/106700eb075d4a7c909afe92fdd0fed2 2024-11-25T03:14:57,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/106700eb075d4a7c909afe92fdd0fed2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:14:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8f40fd47d4cb42768a5f6d3652efba83 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8f40fd47d4cb42768a5f6d3652efba83 2024-11-25T03:14:57,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8f40fd47d4cb42768a5f6d3652efba83, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:14:57,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/0821195674f1491da959f2cef31102d1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/0821195674f1491da959f2cef31102d1 2024-11-25T03:14:57,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/0821195674f1491da959f2cef31102d1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:14:57,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 897f6457507f38ffcd25c9bdd6308219 in 571ms, sequenceid=12, compaction requested=false 2024-11-25T03:14:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:14:57,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T03:14:57,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:57,731 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-25T03:14:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:14:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:14:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:14:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:14:57,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/e17c9813e8c44490be153fd3e176e7f7 is 50, key is test_row_0/A:col10/1732504497229/Put/seqid=0 2024-11-25T03:14:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:14:57,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742073_1249 (size=12001) 2024-11-25T03:14:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:57,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:14:57,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504557810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504557812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504557813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504557812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504557910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504557920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504557920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504557920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:57,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504557920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504558130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504558130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504558130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504558130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,142 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/e17c9813e8c44490be153fd3e176e7f7 2024-11-25T03:14:58,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/866b6e4d49f74a339fc511acd9939198 is 50, key is test_row_0/B:col10/1732504497229/Put/seqid=0 2024-11-25T03:14:58,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742074_1250 (size=12001) 2024-11-25T03:14:58,157 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/866b6e4d49f74a339fc511acd9939198 2024-11-25T03:14:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/4939e79d0f9e43979440a4964377b2c8 is 50, key is test_row_0/C:col10/1732504497229/Put/seqid=0 2024-11-25T03:14:58,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742075_1251 (size=12001) 2024-11-25T03:14:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/4939e79d0f9e43979440a4964377b2c8 2024-11-25T03:14:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/e17c9813e8c44490be153fd3e176e7f7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e17c9813e8c44490be153fd3e176e7f7 2024-11-25T03:14:58,187 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e17c9813e8c44490be153fd3e176e7f7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T03:14:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/866b6e4d49f74a339fc511acd9939198 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/866b6e4d49f74a339fc511acd9939198 2024-11-25T03:14:58,191 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/866b6e4d49f74a339fc511acd9939198, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T03:14:58,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/4939e79d0f9e43979440a4964377b2c8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4939e79d0f9e43979440a4964377b2c8 2024-11-25T03:14:58,197 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4939e79d0f9e43979440a4964377b2c8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T03:14:58,199 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 897f6457507f38ffcd25c9bdd6308219 in 467ms, sequenceid=48, compaction requested=false 2024-11-25T03:14:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:14:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-25T03:14:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-25T03:14:58,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-25T03:14:58,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0880 sec 2024-11-25T03:14:58,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0950 sec 2024-11-25T03:14:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:14:58,250 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:14:58,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:14:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-25T03:14:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:14:58,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:14:58,253 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:14:58,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:14:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:14:58,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T03:14:58,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:58,406 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:14:58,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:14:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:14:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:14:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:14:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/8302199595204df28e31f94722b802c7 is 50, key is test_row_0/A:col10/1732504497804/Put/seqid=0 2024-11-25T03:14:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:14:58,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:14:58,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742076_1252 (size=12001) 2024-11-25T03:14:58,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504498455, max timestamp=1732504498459Num uniques 859; 2024-11-25T03:14:58,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:58,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504498455, max timestamp=1732504498459Num uniques 859; 2024-11-25T03:14:58,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:58,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504498455, max timestamp=1732504498459Num uniques 859; 2024-11-25T03:14:58,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:14:58,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504558479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504558480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504558485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504558486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504558486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:14:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504558590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504558590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504558599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504558600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504558600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504558800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504558800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504558810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504558810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504558810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:58,827 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/8302199595204df28e31f94722b802c7 2024-11-25T03:14:58,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/9c3878075f544fd0a7712a73ad3b485e is 50, key is test_row_0/B:col10/1732504497804/Put/seqid=0 2024-11-25T03:14:58,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742077_1253 (size=12001) 2024-11-25T03:14:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:14:59,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504559110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504559110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504559120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504559120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504559120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,239 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/9c3878075f544fd0a7712a73ad3b485e 2024-11-25T03:14:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/2cf4c6ebfc784b179c38b6f83a9a8340 is 50, key is test_row_0/C:col10/1732504497804/Put/seqid=0 2024-11-25T03:14:59,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742078_1254 (size=12001) 2024-11-25T03:14:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:14:59,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504559619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504559620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504559630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504559630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:14:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504559640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:14:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:14:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:14:59,651 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/2cf4c6ebfc784b179c38b6f83a9a8340 2024-11-25T03:14:59,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/8302199595204df28e31f94722b802c7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/8302199595204df28e31f94722b802c7 2024-11-25T03:14:59,660 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/8302199595204df28e31f94722b802c7, entries=150, sequenceid=59, filesize=11.7 K 2024-11-25T03:14:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/9c3878075f544fd0a7712a73ad3b485e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9c3878075f544fd0a7712a73ad3b485e 2024-11-25T03:14:59,668 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9c3878075f544fd0a7712a73ad3b485e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-25T03:14:59,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/2cf4c6ebfc784b179c38b6f83a9a8340 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/2cf4c6ebfc784b179c38b6f83a9a8340 2024-11-25T03:14:59,673 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/2cf4c6ebfc784b179c38b6f83a9a8340, entries=150, sequenceid=59, filesize=11.7 K 2024-11-25T03:14:59,674 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 897f6457507f38ffcd25c9bdd6308219 in 1268ms, sequenceid=59, compaction requested=true 2024-11-25T03:14:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:14:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:14:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-25T03:14:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-25T03:14:59,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-25T03:14:59,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-25T03:14:59,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4280 sec 2024-11-25T03:15:00,090 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:15:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:15:00,400 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:00,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-25T03:15:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:15:00,402 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:00,403 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:00,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:15:00,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:00,556 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-25T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:00,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:00,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:00,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2a4e56c548574b708b0a5e902e284468 is 50, key is test_row_0/A:col10/1732504498478/Put/seqid=0 2024-11-25T03:15:00,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742079_1255 (size=12001) 2024-11-25T03:15:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:00,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:00,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504560647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504560649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,652 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504560651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504560651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504560659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,664 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:15:00,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504560760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504560760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504560760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,973 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2a4e56c548574b708b0a5e902e284468 2024-11-25T03:15:00,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504560969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504560970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504560970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:00,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/de94e54d54b540a7ade89ebc54aa2e92 is 50, key is test_row_0/B:col10/1732504498478/Put/seqid=0 2024-11-25T03:15:00,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742080_1256 (size=12001) 2024-11-25T03:15:00,989 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/de94e54d54b540a7ade89ebc54aa2e92 2024-11-25T03:15:00,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d9a4b5c11d824899a92f1d116a9dc911 is 50, key is test_row_0/C:col10/1732504498478/Put/seqid=0 2024-11-25T03:15:01,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742081_1257 (size=12001) 2024-11-25T03:15:01,006 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d9a4b5c11d824899a92f1d116a9dc911 2024-11-25T03:15:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2a4e56c548574b708b0a5e902e284468 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2a4e56c548574b708b0a5e902e284468 2024-11-25T03:15:01,016 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2a4e56c548574b708b0a5e902e284468, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T03:15:01,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/de94e54d54b540a7ade89ebc54aa2e92 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/de94e54d54b540a7ade89ebc54aa2e92 2024-11-25T03:15:01,021 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/de94e54d54b540a7ade89ebc54aa2e92, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T03:15:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d9a4b5c11d824899a92f1d116a9dc911 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d9a4b5c11d824899a92f1d116a9dc911 2024-11-25T03:15:01,027 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d9a4b5c11d824899a92f1d116a9dc911, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T03:15:01,028 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 897f6457507f38ffcd25c9bdd6308219 in 472ms, sequenceid=96, compaction requested=true 2024-11-25T03:15:01,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:01,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:01,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-25T03:15:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-25T03:15:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:15:01,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-25T03:15:01,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 627 msec 2024-11-25T03:15:01,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 633 msec 2024-11-25T03:15:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:01,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T03:15:01,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:01,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/69684f4386b5464287561d99a67403f1 is 50, key is test_row_1/A:col10/1732504501282/Put/seqid=0 2024-11-25T03:15:01,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742082_1258 (size=9657) 2024-11-25T03:15:01,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/69684f4386b5464287561d99a67403f1 2024-11-25T03:15:01,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/20734d4ea33e479db352fea34e9c2ba1 is 50, key is test_row_1/B:col10/1732504501282/Put/seqid=0 2024-11-25T03:15:01,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504561310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504561314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504561315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742083_1259 (size=9657) 2024-11-25T03:15:01,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504561420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504561420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504561420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:15:01,540 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:01,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-25T03:15:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:15:01,543 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:01,544 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:01,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504561630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504561630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504561630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:15:01,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:15:01,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:01,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:01,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:01,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:01,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:01,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/20734d4ea33e479db352fea34e9c2ba1 2024-11-25T03:15:01,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/cc7799c4fa0b49cc9424db53c6a3be22 is 50, key is test_row_1/C:col10/1732504501282/Put/seqid=0 2024-11-25T03:15:01,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742084_1260 (size=9657) 2024-11-25T03:15:01,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:15:01,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:01,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:01,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:01,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:01,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:15:01,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:15:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:02,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/cc7799c4fa0b49cc9424db53c6a3be22 2024-11-25T03:15:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/69684f4386b5464287561d99a67403f1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/69684f4386b5464287561d99a67403f1 2024-11-25T03:15:02,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:15:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:02,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/69684f4386b5464287561d99a67403f1, entries=100, sequenceid=109, filesize=9.4 K 2024-11-25T03:15:02,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/20734d4ea33e479db352fea34e9c2ba1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/20734d4ea33e479db352fea34e9c2ba1 2024-11-25T03:15:02,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/20734d4ea33e479db352fea34e9c2ba1, entries=100, sequenceid=109, filesize=9.4 K 2024-11-25T03:15:02,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/cc7799c4fa0b49cc9424db53c6a3be22 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/cc7799c4fa0b49cc9424db53c6a3be22 2024-11-25T03:15:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:15:02,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/cc7799c4fa0b49cc9424db53c6a3be22, entries=100, sequenceid=109, filesize=9.4 K 2024-11-25T03:15:02,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 897f6457507f38ffcd25c9bdd6308219 in 889ms, sequenceid=109, compaction requested=true 2024-11-25T03:15:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:02,173 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T03:15:02,173 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T03:15:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:02,175 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T03:15:02,175 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:02,175 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,175 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8f40fd47d4cb42768a5f6d3652efba83, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/866b6e4d49f74a339fc511acd9939198, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9c3878075f544fd0a7712a73ad3b485e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/de94e54d54b540a7ade89ebc54aa2e92, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/20734d4ea33e479db352fea34e9c2ba1] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=56.3 K 2024-11-25T03:15:02,175 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T03:15:02,176 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:02,176 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,176 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/106700eb075d4a7c909afe92fdd0fed2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e17c9813e8c44490be153fd3e176e7f7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/8302199595204df28e31f94722b802c7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2a4e56c548574b708b0a5e902e284468, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/69684f4386b5464287561d99a67403f1] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=56.3 K 2024-11-25T03:15:02,176 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f40fd47d4cb42768a5f6d3652efba83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732504497123 2024-11-25T03:15:02,177 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106700eb075d4a7c909afe92fdd0fed2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732504497123 2024-11-25T03:15:02,177 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 866b6e4d49f74a339fc511acd9939198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732504497229 2024-11-25T03:15:02,177 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting e17c9813e8c44490be153fd3e176e7f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732504497229 2024-11-25T03:15:02,177 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c3878075f544fd0a7712a73ad3b485e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732504497804 2024-11-25T03:15:02,178 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8302199595204df28e31f94722b802c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732504497804 2024-11-25T03:15:02,178 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting de94e54d54b540a7ade89ebc54aa2e92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732504498478 2024-11-25T03:15:02,178 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4e56c548574b708b0a5e902e284468, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732504498478 2024-11-25T03:15:02,178 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 20734d4ea33e479db352fea34e9c2ba1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504501282 2024-11-25T03:15:02,179 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69684f4386b5464287561d99a67403f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504501282 2024-11-25T03:15:02,191 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:02,191 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:02,192 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/90ecfdbc9ff549688ed5b3a3312570d8 is 50, key is test_row_0/A:col10/1732504498478/Put/seqid=0 2024-11-25T03:15:02,192 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/b704b35c30e64db8997ca168b0caa0ea is 50, key is test_row_0/B:col10/1732504498478/Put/seqid=0 2024-11-25T03:15:02,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742086_1262 (size=12173) 2024-11-25T03:15:02,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742085_1261 (size=12173) 2024-11-25T03:15:02,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:15:02,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,310 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:15:02,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:02,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:02,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/bf1830dea3184973aa2470fa1b51f394 is 50, key is test_row_0/A:col10/1732504501305/Put/seqid=0 2024-11-25T03:15:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742087_1263 (size=12001) 2024-11-25T03:15:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504562467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504562468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504562470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504562580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504562580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504562580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,608 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/b704b35c30e64db8997ca168b0caa0ea as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/b704b35c30e64db8997ca168b0caa0ea 2024-11-25T03:15:02,608 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/90ecfdbc9ff549688ed5b3a3312570d8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/90ecfdbc9ff549688ed5b3a3312570d8 2024-11-25T03:15:02,613 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into b704b35c30e64db8997ca168b0caa0ea(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:02,613 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:02,613 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=11, startTime=1732504502173; duration=0sec 2024-11-25T03:15:02,613 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:02,613 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:02,613 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T03:15:02,615 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T03:15:02,615 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:02,615 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:02,615 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into 90ecfdbc9ff549688ed5b3a3312570d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:02,615 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:02,615 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=11, startTime=1732504502173; duration=0sec 2024-11-25T03:15:02,615 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/0821195674f1491da959f2cef31102d1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4939e79d0f9e43979440a4964377b2c8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/2cf4c6ebfc784b179c38b6f83a9a8340, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d9a4b5c11d824899a92f1d116a9dc911, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/cc7799c4fa0b49cc9424db53c6a3be22] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=56.3 K 2024-11-25T03:15:02,615 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:02,615 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:02,615 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 0821195674f1491da959f2cef31102d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732504497123 2024-11-25T03:15:02,616 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4939e79d0f9e43979440a4964377b2c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732504497229 2024-11-25T03:15:02,616 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf4c6ebfc784b179c38b6f83a9a8340, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732504497804 2024-11-25T03:15:02,616 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a4b5c11d824899a92f1d116a9dc911, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732504498478 2024-11-25T03:15:02,617 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7799c4fa0b49cc9424db53c6a3be22, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504501282 2024-11-25T03:15:02,628 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:02,629 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/83473ca15f314b3a912630f86d54cc13 is 50, key is test_row_0/C:col10/1732504498478/Put/seqid=0 2024-11-25T03:15:02,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742088_1264 (size=12173) 2024-11-25T03:15:02,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504562670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,674 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:15:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504562680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,682 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,721 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/bf1830dea3184973aa2470fa1b51f394 2024-11-25T03:15:02,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/db5a357de98d4b13a8dfb60a4b65f245 is 50, key is test_row_0/B:col10/1732504501305/Put/seqid=0 2024-11-25T03:15:02,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742089_1265 (size=12001) 2024-11-25T03:15:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504562790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504562790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504562790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,038 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/83473ca15f314b3a912630f86d54cc13 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/83473ca15f314b3a912630f86d54cc13 2024-11-25T03:15:03,042 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into 83473ca15f314b3a912630f86d54cc13(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:03,042 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:03,042 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=11, startTime=1732504502173; duration=0sec 2024-11-25T03:15:03,043 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:03,043 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504563100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504563101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504563101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,133 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/db5a357de98d4b13a8dfb60a4b65f245 2024-11-25T03:15:03,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3a5b65607fbd4efbb642fdee22bd501d is 50, key is test_row_0/C:col10/1732504501305/Put/seqid=0 2024-11-25T03:15:03,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742090_1266 (size=12001) 2024-11-25T03:15:03,548 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3a5b65607fbd4efbb642fdee22bd501d 2024-11-25T03:15:03,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/bf1830dea3184973aa2470fa1b51f394 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/bf1830dea3184973aa2470fa1b51f394 2024-11-25T03:15:03,560 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/bf1830dea3184973aa2470fa1b51f394, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:15:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/db5a357de98d4b13a8dfb60a4b65f245 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/db5a357de98d4b13a8dfb60a4b65f245 2024-11-25T03:15:03,566 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/db5a357de98d4b13a8dfb60a4b65f245, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3a5b65607fbd4efbb642fdee22bd501d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3a5b65607fbd4efbb642fdee22bd501d 2024-11-25T03:15:03,572 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3a5b65607fbd4efbb642fdee22bd501d, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:15:03,573 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 897f6457507f38ffcd25c9bdd6308219 in 1263ms, sequenceid=128, compaction requested=false 2024-11-25T03:15:03,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:03,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:03,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-25T03:15:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-25T03:15:03,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-25T03:15:03,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0300 sec 2024-11-25T03:15:03,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.0370 sec 2024-11-25T03:15:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:03,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T03:15:03,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:03,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:03,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:03,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:03,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:03,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:03,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/e0f4d48ad81e4d4f9d34cba643e7cfc3 is 50, key is test_row_0/A:col10/1732504503611/Put/seqid=0 2024-11-25T03:15:03,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742091_1267 (size=12151) 2024-11-25T03:15:03,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/e0f4d48ad81e4d4f9d34cba643e7cfc3 2024-11-25T03:15:03,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/bc615e3af22e4af08fc491b4dc9a601d is 50, key is test_row_0/B:col10/1732504503611/Put/seqid=0 2024-11-25T03:15:03,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742092_1268 (size=12151) 2024-11-25T03:15:03,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504563635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504563636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504563638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:15:03,690 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:03,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-25T03:15:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:15:03,693 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:03,694 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:03,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:03,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504563752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504563752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504563752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:15:03,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:15:03,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:03,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:03,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504563960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504563960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504563960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:15:04,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:04,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:15:04,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/bc615e3af22e4af08fc491b4dc9a601d 2024-11-25T03:15:04,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3ef8b21e827840cc83f579e169a3cc5a is 50, key is test_row_0/C:col10/1732504503611/Put/seqid=0 2024-11-25T03:15:04,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742093_1269 (size=12151) 2024-11-25T03:15:04,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:15:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504564270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504564270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504564270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:15:04,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:04,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:15:04,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3ef8b21e827840cc83f579e169a3cc5a 2024-11-25T03:15:04,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/e0f4d48ad81e4d4f9d34cba643e7cfc3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e0f4d48ad81e4d4f9d34cba643e7cfc3 2024-11-25T03:15:04,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:15:04,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:04,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:04,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e0f4d48ad81e4d4f9d34cba643e7cfc3, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T03:15:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/bc615e3af22e4af08fc491b4dc9a601d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/bc615e3af22e4af08fc491b4dc9a601d 2024-11-25T03:15:04,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/bc615e3af22e4af08fc491b4dc9a601d, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T03:15:04,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3ef8b21e827840cc83f579e169a3cc5a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3ef8b21e827840cc83f579e169a3cc5a 2024-11-25T03:15:04,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3ef8b21e827840cc83f579e169a3cc5a, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T03:15:04,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 897f6457507f38ffcd25c9bdd6308219 in 864ms, sequenceid=146, compaction requested=true 2024-11-25T03:15:04,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:04,478 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:04,478 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:04,479 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36325 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:04,479 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:04,479 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36325 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:04,479 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:04,479 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,479 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,479 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/90ecfdbc9ff549688ed5b3a3312570d8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/bf1830dea3184973aa2470fa1b51f394, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e0f4d48ad81e4d4f9d34cba643e7cfc3] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=35.5 K 2024-11-25T03:15:04,479 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/b704b35c30e64db8997ca168b0caa0ea, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/db5a357de98d4b13a8dfb60a4b65f245, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/bc615e3af22e4af08fc491b4dc9a601d] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=35.5 K 2024-11-25T03:15:04,479 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90ecfdbc9ff549688ed5b3a3312570d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504498478 2024-11-25T03:15:04,480 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b704b35c30e64db8997ca168b0caa0ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504498478 2024-11-25T03:15:04,480 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1830dea3184973aa2470fa1b51f394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504501305 2024-11-25T03:15:04,480 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting db5a357de98d4b13a8dfb60a4b65f245, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504501305 2024-11-25T03:15:04,480 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f4d48ad81e4d4f9d34cba643e7cfc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732504502466 2024-11-25T03:15:04,480 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting bc615e3af22e4af08fc491b4dc9a601d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732504502466 2024-11-25T03:15:04,490 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:04,490 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/d93a0f7705914255b7e24fda4afd830c is 50, key is test_row_0/B:col10/1732504503611/Put/seqid=0 2024-11-25T03:15:04,491 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:04,492 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6425bd32bfbf403eb425783ec8c6ba0e is 50, key is test_row_0/A:col10/1732504503611/Put/seqid=0 2024-11-25T03:15:04,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742094_1270 (size=12425) 2024-11-25T03:15:04,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742095_1271 (size=12425) 2024-11-25T03:15:04,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:15:04,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,618 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:15:04,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:04,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:04,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:04,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:04,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:04,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:04,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/12259611e64148dc93136a9d31e479cd is 50, key is test_row_0/A:col10/1732504503636/Put/seqid=0 2024-11-25T03:15:04,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742096_1272 (size=12151) 2024-11-25T03:15:04,647 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/12259611e64148dc93136a9d31e479cd 2024-11-25T03:15:04,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/ebd53a7b43ee4ba4a5b66113746de762 is 50, key is test_row_0/B:col10/1732504503636/Put/seqid=0 2024-11-25T03:15:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742097_1273 (size=12151) 2024-11-25T03:15:04,756 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T03:15:04,756 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T03:15:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:04,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:04,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504564812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504564814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504564815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:15:04,904 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/d93a0f7705914255b7e24fda4afd830c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/d93a0f7705914255b7e24fda4afd830c 2024-11-25T03:15:04,909 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6425bd32bfbf403eb425783ec8c6ba0e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6425bd32bfbf403eb425783ec8c6ba0e 2024-11-25T03:15:04,910 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into d93a0f7705914255b7e24fda4afd830c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:04,910 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:04,910 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=13, startTime=1732504504478; duration=0sec 2024-11-25T03:15:04,910 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:04,910 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:04,910 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:04,911 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36325 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:04,911 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:04,911 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:04,911 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/83473ca15f314b3a912630f86d54cc13, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3a5b65607fbd4efbb642fdee22bd501d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3ef8b21e827840cc83f579e169a3cc5a] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=35.5 K 2024-11-25T03:15:04,912 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 83473ca15f314b3a912630f86d54cc13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504498478 2024-11-25T03:15:04,912 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a5b65607fbd4efbb642fdee22bd501d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504501305 2024-11-25T03:15:04,913 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef8b21e827840cc83f579e169a3cc5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732504502466 2024-11-25T03:15:04,915 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into 6425bd32bfbf403eb425783ec8c6ba0e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:04,915 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:04,915 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=13, startTime=1732504504477; duration=0sec 2024-11-25T03:15:04,915 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:04,915 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:04,921 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:04,922 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/52ad47599676416dae1ad2c50cf86aa1 is 50, key is test_row_0/C:col10/1732504503611/Put/seqid=0 2024-11-25T03:15:04,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504564920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504564921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:04,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742098_1274 (size=12425) 2024-11-25T03:15:04,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504564930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:04,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:04,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,067 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/ebd53a7b43ee4ba4a5b66113746de762 2024-11-25T03:15:05,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3d781f060e38497cbdd25ed6f710bb2a is 50, key is test_row_0/C:col10/1732504503636/Put/seqid=0 2024-11-25T03:15:05,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742099_1275 (size=12151) 2024-11-25T03:15:05,081 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3d781f060e38497cbdd25ed6f710bb2a 2024-11-25T03:15:05,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/12259611e64148dc93136a9d31e479cd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/12259611e64148dc93136a9d31e479cd 2024-11-25T03:15:05,090 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/12259611e64148dc93136a9d31e479cd, entries=150, sequenceid=163, filesize=11.9 K 2024-11-25T03:15:05,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/ebd53a7b43ee4ba4a5b66113746de762 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ebd53a7b43ee4ba4a5b66113746de762 2024-11-25T03:15:05,094 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ebd53a7b43ee4ba4a5b66113746de762, entries=150, sequenceid=163, filesize=11.9 K 2024-11-25T03:15:05,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/3d781f060e38497cbdd25ed6f710bb2a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3d781f060e38497cbdd25ed6f710bb2a 2024-11-25T03:15:05,100 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3d781f060e38497cbdd25ed6f710bb2a, entries=150, sequenceid=163, filesize=11.9 K 2024-11-25T03:15:05,101 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 897f6457507f38ffcd25c9bdd6308219 in 483ms, sequenceid=163, compaction requested=false 2024-11-25T03:15:05,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:05,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:05,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-25T03:15:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-25T03:15:05,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-25T03:15:05,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-25T03:15:05,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4140 sec 2024-11-25T03:15:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:05,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:05,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/acde60b20b5242e98c36ff9bdd1fd0cd is 50, key is test_row_0/A:col10/1732504504804/Put/seqid=0 2024-11-25T03:15:05,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742100_1276 (size=12151) 2024-11-25T03:15:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504565162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504565166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504565167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504565280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504565280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504565281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,336 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/52ad47599676416dae1ad2c50cf86aa1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/52ad47599676416dae1ad2c50cf86aa1 2024-11-25T03:15:05,341 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into 52ad47599676416dae1ad2c50cf86aa1(size=12.1 K), total size for store is 24 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:05,341 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:05,341 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=13, startTime=1732504504478; duration=0sec 2024-11-25T03:15:05,341 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:05,341 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504565490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504565490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504565490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/acde60b20b5242e98c36ff9bdd1fd0cd 2024-11-25T03:15:05,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/c04bb7173ceb4b3e94b623a21e8f31a1 is 50, key is test_row_0/B:col10/1732504504804/Put/seqid=0 2024-11-25T03:15:05,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742101_1277 (size=12151) 2024-11-25T03:15:05,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/c04bb7173ceb4b3e94b623a21e8f31a1 2024-11-25T03:15:05,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/919b10baa3c248158f52c8fbb4e483a2 is 50, key is test_row_0/C:col10/1732504504804/Put/seqid=0 2024-11-25T03:15:05,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742102_1278 (size=12151) 2024-11-25T03:15:05,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504565800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504565800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504565800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:15:05,840 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:05,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-25T03:15:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:15:05,842 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:05,843 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:05,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:15:05,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/919b10baa3c248158f52c8fbb4e483a2 2024-11-25T03:15:05,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/acde60b20b5242e98c36ff9bdd1fd0cd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/acde60b20b5242e98c36ff9bdd1fd0cd 2024-11-25T03:15:05,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/acde60b20b5242e98c36ff9bdd1fd0cd, entries=150, sequenceid=182, filesize=11.9 K 2024-11-25T03:15:05,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/c04bb7173ceb4b3e94b623a21e8f31a1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/c04bb7173ceb4b3e94b623a21e8f31a1 2024-11-25T03:15:05,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/c04bb7173ceb4b3e94b623a21e8f31a1, entries=150, sequenceid=182, filesize=11.9 K 2024-11-25T03:15:05,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/919b10baa3c248158f52c8fbb4e483a2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/919b10baa3c248158f52c8fbb4e483a2 2024-11-25T03:15:05,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-25T03:15:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:05,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:05,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/919b10baa3c248158f52c8fbb4e483a2, entries=150, sequenceid=182, filesize=11.9 K 2024-11-25T03:15:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:05,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 897f6457507f38ffcd25c9bdd6308219 in 863ms, sequenceid=182, compaction requested=true 2024-11-25T03:15:05,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:05,997 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:05,997 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:05,998 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:05,998 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:05,998 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:05,998 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:05,998 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6425bd32bfbf403eb425783ec8c6ba0e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/12259611e64148dc93136a9d31e479cd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/acde60b20b5242e98c36ff9bdd1fd0cd] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=35.9 K 2024-11-25T03:15:05,998 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:05,998 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:05,998 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/d93a0f7705914255b7e24fda4afd830c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ebd53a7b43ee4ba4a5b66113746de762, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/c04bb7173ceb4b3e94b623a21e8f31a1] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=35.9 K 2024-11-25T03:15:05,998 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6425bd32bfbf403eb425783ec8c6ba0e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732504502466 2024-11-25T03:15:05,999 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d93a0f7705914255b7e24fda4afd830c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732504502466 2024-11-25T03:15:05,999 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12259611e64148dc93136a9d31e479cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732504503634 2024-11-25T03:15:05,999 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd53a7b43ee4ba4a5b66113746de762, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732504503634 2024-11-25T03:15:05,999 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting acde60b20b5242e98c36ff9bdd1fd0cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732504504790 2024-11-25T03:15:05,999 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting c04bb7173ceb4b3e94b623a21e8f31a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732504504790 2024-11-25T03:15:06,010 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:06,010 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:06,011 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/9b3276aecd0044aab2e38172308ac7e4 is 50, key is test_row_0/B:col10/1732504504804/Put/seqid=0 2024-11-25T03:15:06,011 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/b5b793af0a294852a2a1da91b67a3a4e is 50, key is test_row_0/A:col10/1732504504804/Put/seqid=0 2024-11-25T03:15:06,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742103_1279 (size=12527) 2024-11-25T03:15:06,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742104_1280 (size=12527) 2024-11-25T03:15:06,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-25T03:15:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:06,149 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T03:15:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/10f9c5ca4ce446f392ec38756db0395e is 50, key is test_row_0/A:col10/1732504505165/Put/seqid=0 2024-11-25T03:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:15:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742105_1281 (size=12151) 2024-11-25T03:15:06,170 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/10f9c5ca4ce446f392ec38756db0395e 2024-11-25T03:15:06,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8dafcb074a59495aa9ac61cfac653391 is 50, key is test_row_0/B:col10/1732504505165/Put/seqid=0 2024-11-25T03:15:06,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742106_1282 (size=12151) 2024-11-25T03:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:06,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504566332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504566333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504566335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,446 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/b5b793af0a294852a2a1da91b67a3a4e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b5b793af0a294852a2a1da91b67a3a4e 2024-11-25T03:15:06,450 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/9b3276aecd0044aab2e38172308ac7e4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9b3276aecd0044aab2e38172308ac7e4 2024-11-25T03:15:06,452 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into b5b793af0a294852a2a1da91b67a3a4e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:06,453 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:06,453 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=13, startTime=1732504505997; duration=0sec 2024-11-25T03:15:06,453 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:06,453 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:06,453 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:06,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504566450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,454 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:06,454 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:06,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,454 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504566450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,455 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/52ad47599676416dae1ad2c50cf86aa1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3d781f060e38497cbdd25ed6f710bb2a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/919b10baa3c248158f52c8fbb4e483a2] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=35.9 K 2024-11-25T03:15:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504566450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,455 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ad47599676416dae1ad2c50cf86aa1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732504502466 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,456 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d781f060e38497cbdd25ed6f710bb2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732504503634 2024-11-25T03:15:06,456 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into 9b3276aecd0044aab2e38172308ac7e4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:06,456 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:06,456 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=13, startTime=1732504505997; duration=0sec 2024-11-25T03:15:06,456 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 919b10baa3c248158f52c8fbb4e483a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732504504790 2024-11-25T03:15:06,456 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:06,456 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:06,465 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:06,465 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/61647c0c9ee54521baa47aa7f5d109e4 is 50, key is test_row_0/C:col10/1732504504804/Put/seqid=0 2024-11-25T03:15:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:15:06,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742107_1283 (size=12527) 2024-11-25T03:15:06,590 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8dafcb074a59495aa9ac61cfac653391 2024-11-25T03:15:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/6e362c70d41c4cabb10d903d01986319 is 50, key is test_row_0/C:col10/1732504505165/Put/seqid=0 2024-11-25T03:15:06,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742108_1284 (size=12151) 2024-11-25T03:15:06,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504566660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504566660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504566660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504566690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,693 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504566710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,714 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,875 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/61647c0c9ee54521baa47aa7f5d109e4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/61647c0c9ee54521baa47aa7f5d109e4 2024-11-25T03:15:06,880 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into 61647c0c9ee54521baa47aa7f5d109e4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:06,880 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:06,880 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=13, startTime=1732504505997; duration=0sec 2024-11-25T03:15:06,880 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:06,880 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:06,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504566970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504566970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504566970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:15:07,003 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/6e362c70d41c4cabb10d903d01986319 2024-11-25T03:15:07,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/10f9c5ca4ce446f392ec38756db0395e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10f9c5ca4ce446f392ec38756db0395e 2024-11-25T03:15:07,011 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10f9c5ca4ce446f392ec38756db0395e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-25T03:15:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8dafcb074a59495aa9ac61cfac653391 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8dafcb074a59495aa9ac61cfac653391 2024-11-25T03:15:07,017 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8dafcb074a59495aa9ac61cfac653391, entries=150, sequenceid=199, filesize=11.9 K 2024-11-25T03:15:07,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/6e362c70d41c4cabb10d903d01986319 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6e362c70d41c4cabb10d903d01986319 2024-11-25T03:15:07,022 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6e362c70d41c4cabb10d903d01986319, entries=150, sequenceid=199, filesize=11.9 K 2024-11-25T03:15:07,023 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 897f6457507f38ffcd25c9bdd6308219 in 874ms, sequenceid=199, compaction requested=false 2024-11-25T03:15:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-25T03:15:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-25T03:15:07,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-25T03:15:07,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1820 sec 2024-11-25T03:15:07,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1870 sec 2024-11-25T03:15:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:07,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:15:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:07,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/fce55a26eb01413b842736df0ffedd8e is 50, key is test_row_0/A:col10/1732504506331/Put/seqid=0 2024-11-25T03:15:07,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742109_1285 (size=14541) 2024-11-25T03:15:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504567530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504567531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504567534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504567640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504567640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504567650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504567850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504567850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504567860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:07,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/fce55a26eb01413b842736df0ffedd8e 2024-11-25T03:15:07,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/79b3b6e50fc941458a1c9b42d6b8bd9d is 50, key is test_row_0/B:col10/1732504506331/Put/seqid=0 2024-11-25T03:15:07,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742110_1286 (size=12151) 2024-11-25T03:15:07,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/79b3b6e50fc941458a1c9b42d6b8bd9d 2024-11-25T03:15:07,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/f5fb2f6ed3884702ac0cb736ced13d8f is 50, key is test_row_0/C:col10/1732504506331/Put/seqid=0 2024-11-25T03:15:07,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742111_1287 (size=12151) 2024-11-25T03:15:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:15:07,991 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:07,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-25T03:15:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T03:15:07,993 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:07,993 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:07,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T03:15:08,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-25T03:15:08,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:08,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504568160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504568160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504568170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-25T03:15:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T03:15:08,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/f5fb2f6ed3884702ac0cb736ced13d8f 2024-11-25T03:15:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/fce55a26eb01413b842736df0ffedd8e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/fce55a26eb01413b842736df0ffedd8e 2024-11-25T03:15:08,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/fce55a26eb01413b842736df0ffedd8e, entries=200, sequenceid=221, filesize=14.2 K 2024-11-25T03:15:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/79b3b6e50fc941458a1c9b42d6b8bd9d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/79b3b6e50fc941458a1c9b42d6b8bd9d 2024-11-25T03:15:08,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/79b3b6e50fc941458a1c9b42d6b8bd9d, entries=150, sequenceid=221, filesize=11.9 K 2024-11-25T03:15:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/f5fb2f6ed3884702ac0cb736ced13d8f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f5fb2f6ed3884702ac0cb736ced13d8f 2024-11-25T03:15:08,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f5fb2f6ed3884702ac0cb736ced13d8f, entries=150, sequenceid=221, filesize=11.9 K 2024-11-25T03:15:08,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 897f6457507f38ffcd25c9bdd6308219 in 859ms, sequenceid=221, compaction requested=true 2024-11-25T03:15:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:08,343 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:08,343 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:08,345 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:08,345 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:08,345 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:08,345 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,345 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:08,345 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b5b793af0a294852a2a1da91b67a3a4e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10f9c5ca4ce446f392ec38756db0395e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/fce55a26eb01413b842736df0ffedd8e] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=38.3 K 2024-11-25T03:15:08,345 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,345 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9b3276aecd0044aab2e38172308ac7e4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8dafcb074a59495aa9ac61cfac653391, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/79b3b6e50fc941458a1c9b42d6b8bd9d] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.0 K 2024-11-25T03:15:08,345 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b793af0a294852a2a1da91b67a3a4e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732504504790 2024-11-25T03:15:08,346 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3276aecd0044aab2e38172308ac7e4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732504504790 2024-11-25T03:15:08,346 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 10f9c5ca4ce446f392ec38756db0395e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732504505154 2024-11-25T03:15:08,346 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dafcb074a59495aa9ac61cfac653391, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732504505154 2024-11-25T03:15:08,346 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting fce55a26eb01413b842736df0ffedd8e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732504506331 2024-11-25T03:15:08,347 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b3b6e50fc941458a1c9b42d6b8bd9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732504506331 2024-11-25T03:15:08,357 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:08,357 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:08,357 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/62c26fcccccf4e0fa0df3903c9d21193 is 50, key is test_row_0/A:col10/1732504506331/Put/seqid=0 2024-11-25T03:15:08,358 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/05efced1be564b55b9de195b14e98115 is 50, key is test_row_0/B:col10/1732504506331/Put/seqid=0 2024-11-25T03:15:08,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742113_1289 (size=12629) 2024-11-25T03:15:08,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742112_1288 (size=12629) 2024-11-25T03:15:08,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-25T03:15:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,454 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T03:15:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:08,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:08,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:08,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6eb71d0e53f3476ca6bcf6aad3d212bc is 50, key is test_row_0/A:col10/1732504507533/Put/seqid=0 2024-11-25T03:15:08,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742114_1290 (size=12151) 2024-11-25T03:15:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T03:15:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:08,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504568702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504568703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504568704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,777 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/62c26fcccccf4e0fa0df3903c9d21193 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/62c26fcccccf4e0fa0df3903c9d21193 2024-11-25T03:15:08,787 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/05efced1be564b55b9de195b14e98115 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/05efced1be564b55b9de195b14e98115 2024-11-25T03:15:08,787 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into 62c26fcccccf4e0fa0df3903c9d21193(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:08,787 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:08,787 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=13, startTime=1732504508343; duration=0sec 2024-11-25T03:15:08,787 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:08,787 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:08,787 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:08,788 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:08,789 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:08,789 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:08,789 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/61647c0c9ee54521baa47aa7f5d109e4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6e362c70d41c4cabb10d903d01986319, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f5fb2f6ed3884702ac0cb736ced13d8f] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.0 K 2024-11-25T03:15:08,789 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 61647c0c9ee54521baa47aa7f5d109e4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732504504790 2024-11-25T03:15:08,790 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e362c70d41c4cabb10d903d01986319, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732504505154 2024-11-25T03:15:08,791 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f5fb2f6ed3884702ac0cb736ced13d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732504506331 2024-11-25T03:15:08,792 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into 05efced1be564b55b9de195b14e98115(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:08,792 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:08,792 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=13, startTime=1732504508343; duration=0sec 2024-11-25T03:15:08,792 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:08,792 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:08,799 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:08,799 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/27a2ac33d6384edb9d430aacebf1acc6 is 50, key is test_row_0/C:col10/1732504506331/Put/seqid=0 2024-11-25T03:15:08,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742115_1291 (size=12629) 2024-11-25T03:15:08,808 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/27a2ac33d6384edb9d430aacebf1acc6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/27a2ac33d6384edb9d430aacebf1acc6 2024-11-25T03:15:08,813 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into 27a2ac33d6384edb9d430aacebf1acc6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:08,813 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:08,813 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=13, startTime=1732504508343; duration=0sec 2024-11-25T03:15:08,813 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:08,813 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:08,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504568810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504568810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504568820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:08,863 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6eb71d0e53f3476ca6bcf6aad3d212bc 2024-11-25T03:15:08,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/7ddf083b881546c5b84d7f517134d814 is 50, key is test_row_0/B:col10/1732504507533/Put/seqid=0 2024-11-25T03:15:08,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742116_1292 (size=12151) 2024-11-25T03:15:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504569020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504569020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504569030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T03:15:09,283 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/7ddf083b881546c5b84d7f517134d814 2024-11-25T03:15:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/5ece1447db554a369d20c2f8dc4c16ba is 50, key is test_row_0/C:col10/1732504507533/Put/seqid=0 2024-11-25T03:15:09,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742117_1293 (size=12151) 2024-11-25T03:15:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504569330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504569330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504569340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,711 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/5ece1447db554a369d20c2f8dc4c16ba 2024-11-25T03:15:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6eb71d0e53f3476ca6bcf6aad3d212bc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6eb71d0e53f3476ca6bcf6aad3d212bc 2024-11-25T03:15:09,720 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6eb71d0e53f3476ca6bcf6aad3d212bc, entries=150, sequenceid=234, filesize=11.9 K 2024-11-25T03:15:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/7ddf083b881546c5b84d7f517134d814 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/7ddf083b881546c5b84d7f517134d814 2024-11-25T03:15:09,725 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/7ddf083b881546c5b84d7f517134d814, entries=150, sequenceid=234, filesize=11.9 K 2024-11-25T03:15:09,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/5ece1447db554a369d20c2f8dc4c16ba as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/5ece1447db554a369d20c2f8dc4c16ba 2024-11-25T03:15:09,730 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/5ece1447db554a369d20c2f8dc4c16ba, entries=150, sequenceid=234, filesize=11.9 K 2024-11-25T03:15:09,731 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 897f6457507f38ffcd25c9bdd6308219 in 1277ms, sequenceid=234, compaction requested=false 2024-11-25T03:15:09,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:09,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:09,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-25T03:15:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-25T03:15:09,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-25T03:15:09,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-25T03:15:09,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7440 sec 2024-11-25T03:15:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:09,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-25T03:15:09,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:09,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6efd1131b4c442b0b67707c1501c4396 is 50, key is test_row_0/A:col10/1732504508701/Put/seqid=0 2024-11-25T03:15:09,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742118_1294 (size=17031) 2024-11-25T03:15:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504569859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504569860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504569860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504569970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504569970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504569970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T03:15:10,141 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:10,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-25T03:15:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:15:10,144 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:10,144 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:10,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504570180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504570180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504570180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:15:10,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6efd1131b4c442b0b67707c1501c4396 2024-11-25T03:15:10,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/74f09574c5a841878bc02cb9845c7fbb is 50, key is test_row_0/B:col10/1732504508701/Put/seqid=0 2024-11-25T03:15:10,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742119_1295 (size=12251) 2024-11-25T03:15:10,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-25T03:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-25T03:15:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:15:10,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504570489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504570489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504570490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:10,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-25T03:15:10,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:10,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/74f09574c5a841878bc02cb9845c7fbb 2024-11-25T03:15:10,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/436f3324e09149bcbc7b10529f318cac is 50, key is test_row_0/C:col10/1732504508701/Put/seqid=0 2024-11-25T03:15:10,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742120_1296 (size=12251) 2024-11-25T03:15:10,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-25T03:15:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:15:10,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-25T03:15:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:10,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:11,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504570999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504571000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504571000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:11,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-25T03:15:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:11,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:11,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/436f3324e09149bcbc7b10529f318cac 2024-11-25T03:15:11,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6efd1131b4c442b0b67707c1501c4396 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6efd1131b4c442b0b67707c1501c4396 2024-11-25T03:15:11,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6efd1131b4c442b0b67707c1501c4396, entries=250, sequenceid=258, filesize=16.6 K 2024-11-25T03:15:11,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/74f09574c5a841878bc02cb9845c7fbb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/74f09574c5a841878bc02cb9845c7fbb 2024-11-25T03:15:11,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/74f09574c5a841878bc02cb9845c7fbb, entries=150, sequenceid=258, filesize=12.0 K 2024-11-25T03:15:11,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/436f3324e09149bcbc7b10529f318cac as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/436f3324e09149bcbc7b10529f318cac 2024-11-25T03:15:11,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/436f3324e09149bcbc7b10529f318cac, entries=150, sequenceid=258, filesize=12.0 K 2024-11-25T03:15:11,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 897f6457507f38ffcd25c9bdd6308219 in 1258ms, sequenceid=258, compaction requested=true 2024-11-25T03:15:11,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:11,101 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:11,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:11,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:11,101 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:11,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:11,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:11,102 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41811 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:11,102 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:11,102 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:11,102 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/62c26fcccccf4e0fa0df3903c9d21193, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6eb71d0e53f3476ca6bcf6aad3d212bc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6efd1131b4c442b0b67707c1501c4396] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=40.8 K 2024-11-25T03:15:11,102 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:11,102 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:11,103 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:11,103 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/05efced1be564b55b9de195b14e98115, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/7ddf083b881546c5b84d7f517134d814, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/74f09574c5a841878bc02cb9845c7fbb] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.2 K 2024-11-25T03:15:11,103 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c26fcccccf4e0fa0df3903c9d21193, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732504506331 2024-11-25T03:15:11,104 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 05efced1be564b55b9de195b14e98115, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732504506331 2024-11-25T03:15:11,104 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eb71d0e53f3476ca6bcf6aad3d212bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732504507528 2024-11-25T03:15:11,104 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6efd1131b4c442b0b67707c1501c4396, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732504508694 2024-11-25T03:15:11,104 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ddf083b881546c5b84d7f517134d814, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732504507528 2024-11-25T03:15:11,105 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f09574c5a841878bc02cb9845c7fbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732504508701 2024-11-25T03:15:11,112 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:11,113 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/e727d2078c3e4360ba69a4e7bd5287c5 is 50, key is test_row_0/B:col10/1732504508701/Put/seqid=0 2024-11-25T03:15:11,113 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:11,113 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6db3201742574a5c96d83a890225b24b is 50, key is test_row_0/A:col10/1732504508701/Put/seqid=0 2024-11-25T03:15:11,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742122_1298 (size=12831) 2024-11-25T03:15:11,125 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6db3201742574a5c96d83a890225b24b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6db3201742574a5c96d83a890225b24b 2024-11-25T03:15:11,130 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into 6db3201742574a5c96d83a890225b24b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:11,131 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:11,131 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=13, startTime=1732504511101; duration=0sec 2024-11-25T03:15:11,131 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:11,131 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:11,131 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:11,132 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:11,132 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:11,132 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:11,132 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/27a2ac33d6384edb9d430aacebf1acc6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/5ece1447db554a369d20c2f8dc4c16ba, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/436f3324e09149bcbc7b10529f318cac] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.2 K 2024-11-25T03:15:11,133 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27a2ac33d6384edb9d430aacebf1acc6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732504506331 2024-11-25T03:15:11,133 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ece1447db554a369d20c2f8dc4c16ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732504507528 2024-11-25T03:15:11,134 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 436f3324e09149bcbc7b10529f318cac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732504508701 2024-11-25T03:15:11,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742121_1297 (size=12831) 2024-11-25T03:15:11,142 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:11,143 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/f71e084b004f4d5291131dc56e7f60d1 is 50, key is test_row_0/C:col10/1732504508701/Put/seqid=0 2024-11-25T03:15:11,143 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/e727d2078c3e4360ba69a4e7bd5287c5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/e727d2078c3e4360ba69a4e7bd5287c5 2024-11-25T03:15:11,149 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into e727d2078c3e4360ba69a4e7bd5287c5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:11,149 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:11,149 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=13, startTime=1732504511101; duration=0sec 2024-11-25T03:15:11,149 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:11,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742123_1299 (size=12831) 2024-11-25T03:15:11,149 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:11,159 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/f71e084b004f4d5291131dc56e7f60d1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f71e084b004f4d5291131dc56e7f60d1 2024-11-25T03:15:11,166 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into f71e084b004f4d5291131dc56e7f60d1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:11,167 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:11,167 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=13, startTime=1732504511102; duration=0sec 2024-11-25T03:15:11,167 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:11,167 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:11,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-25T03:15:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:11,218 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:15:11,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:11,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:11,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:11,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:11,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:11,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/d702d098ebcc4ec580f6512d55226d4e is 50, key is test_row_0/A:col10/1732504509851/Put/seqid=0 2024-11-25T03:15:11,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742124_1300 (size=12301) 2024-11-25T03:15:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:15:11,631 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/d702d098ebcc4ec580f6512d55226d4e 2024-11-25T03:15:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/4a49c70769ab49f6a86df0c6f29bdd45 is 50, key is test_row_0/B:col10/1732504509851/Put/seqid=0 2024-11-25T03:15:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742125_1301 (size=12301) 2024-11-25T03:15:11,649 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/4a49c70769ab49f6a86df0c6f29bdd45 2024-11-25T03:15:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/269d7706a78642d2afacd03daba92bac is 50, key is test_row_0/C:col10/1732504509851/Put/seqid=0 2024-11-25T03:15:11,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742126_1302 (size=12301) 2024-11-25T03:15:11,663 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/269d7706a78642d2afacd03daba92bac 2024-11-25T03:15:11,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/d702d098ebcc4ec580f6512d55226d4e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d702d098ebcc4ec580f6512d55226d4e 2024-11-25T03:15:11,677 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d702d098ebcc4ec580f6512d55226d4e, entries=150, sequenceid=272, filesize=12.0 K 2024-11-25T03:15:11,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/4a49c70769ab49f6a86df0c6f29bdd45 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/4a49c70769ab49f6a86df0c6f29bdd45 2024-11-25T03:15:11,685 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/4a49c70769ab49f6a86df0c6f29bdd45, entries=150, sequenceid=272, filesize=12.0 K 2024-11-25T03:15:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/269d7706a78642d2afacd03daba92bac as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/269d7706a78642d2afacd03daba92bac 2024-11-25T03:15:11,690 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/269d7706a78642d2afacd03daba92bac, entries=150, sequenceid=272, filesize=12.0 K 2024-11-25T03:15:11,693 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 897f6457507f38ffcd25c9bdd6308219 in 475ms, sequenceid=272, compaction requested=false 2024-11-25T03:15:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-25T03:15:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-25T03:15:11,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-25T03:15:11,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-11-25T03:15:11,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.5590 sec 2024-11-25T03:15:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:12,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:15:12,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:12,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:12,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:12,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:12,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:12,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:12,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/10b53ba3a7d14dca96514be6b6714bcc is 50, key is test_row_0/A:col10/1732504512019/Put/seqid=0 2024-11-25T03:15:12,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742127_1303 (size=14741) 2024-11-25T03:15:12,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/10b53ba3a7d14dca96514be6b6714bcc 2024-11-25T03:15:12,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/3efdae32647047efb3a3f71dc9b452a6 is 50, key is test_row_0/B:col10/1732504512019/Put/seqid=0 2024-11-25T03:15:12,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742128_1304 (size=12301) 2024-11-25T03:15:12,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504512053, max timestamp=1732504512057Num uniques 859; 2024-11-25T03:15:12,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504512053, max timestamp=1732504512057Num uniques 859; 2024-11-25T03:15:12,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:12,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:12,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504512053, max timestamp=1732504512057Num uniques 859; 2024-11-25T03:15:12,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:12,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504572090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504572091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504572098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504572210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504572210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504572210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:15:12,290 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:12,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-25T03:15:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-25T03:15:12,293 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:12,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:12,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-25T03:15:12,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504572420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504572421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504572421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-25T03:15:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/3efdae32647047efb3a3f71dc9b452a6 2024-11-25T03:15:12,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/6fd2b09b2e094cf0998c420c1efc5d20 is 50, key is test_row_0/C:col10/1732504512019/Put/seqid=0 2024-11-25T03:15:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742129_1305 (size=12301) 2024-11-25T03:15:12,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-25T03:15:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-25T03:15:12,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504572730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504572730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504572730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:12,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-25T03:15:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:12,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/6fd2b09b2e094cf0998c420c1efc5d20 2024-11-25T03:15:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/10b53ba3a7d14dca96514be6b6714bcc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10b53ba3a7d14dca96514be6b6714bcc 2024-11-25T03:15:12,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10b53ba3a7d14dca96514be6b6714bcc, entries=200, sequenceid=283, filesize=14.4 K 2024-11-25T03:15:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/3efdae32647047efb3a3f71dc9b452a6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/3efdae32647047efb3a3f71dc9b452a6 2024-11-25T03:15:12,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/3efdae32647047efb3a3f71dc9b452a6, entries=150, sequenceid=283, filesize=12.0 K 2024-11-25T03:15:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/6fd2b09b2e094cf0998c420c1efc5d20 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6fd2b09b2e094cf0998c420c1efc5d20 2024-11-25T03:15:12,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6fd2b09b2e094cf0998c420c1efc5d20, entries=150, sequenceid=283, filesize=12.0 K 2024-11-25T03:15:12,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 897f6457507f38ffcd25c9bdd6308219 in 859ms, sequenceid=283, compaction requested=true 2024-11-25T03:15:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:12,883 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:12,883 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:12,885 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:12,885 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:12,885 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,886 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6db3201742574a5c96d83a890225b24b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d702d098ebcc4ec580f6512d55226d4e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10b53ba3a7d14dca96514be6b6714bcc] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=38.9 K 2024-11-25T03:15:12,886 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:12,886 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:12,886 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db3201742574a5c96d83a890225b24b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732504508701 2024-11-25T03:15:12,886 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,886 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/e727d2078c3e4360ba69a4e7bd5287c5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/4a49c70769ab49f6a86df0c6f29bdd45, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/3efdae32647047efb3a3f71dc9b452a6] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.6 K 2024-11-25T03:15:12,886 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d702d098ebcc4ec580f6512d55226d4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732504509851 2024-11-25T03:15:12,887 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e727d2078c3e4360ba69a4e7bd5287c5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732504508701 2024-11-25T03:15:12,887 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b53ba3a7d14dca96514be6b6714bcc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732504512019 2024-11-25T03:15:12,887 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a49c70769ab49f6a86df0c6f29bdd45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732504509851 2024-11-25T03:15:12,887 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 3efdae32647047efb3a3f71dc9b452a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732504512019 2024-11-25T03:15:12,894 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:12,895 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2d6b654aafc44ce3b80bddb10828dee3 is 50, key is test_row_0/A:col10/1732504512019/Put/seqid=0 2024-11-25T03:15:12,897 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:12,898 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/f2e28e090ff64d85b9d1827b68edd0c2 is 50, key is test_row_0/B:col10/1732504512019/Put/seqid=0 2024-11-25T03:15:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742130_1306 (size=12983) 2024-11-25T03:15:12,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742131_1307 (size=12983) 2024-11-25T03:15:12,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-25T03:15:12,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:12,907 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-25T03:15:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:12,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/f3aad8893c104560b770456a502668b6 is 50, key is test_row_0/A:col10/1732504512090/Put/seqid=0 2024-11-25T03:15:12,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742132_1308 (size=12301) 2024-11-25T03:15:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-25T03:15:13,187 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:15:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504573252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504573256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504573260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,306 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2d6b654aafc44ce3b80bddb10828dee3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2d6b654aafc44ce3b80bddb10828dee3 2024-11-25T03:15:13,308 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/f2e28e090ff64d85b9d1827b68edd0c2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/f2e28e090ff64d85b9d1827b68edd0c2 2024-11-25T03:15:13,311 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into 2d6b654aafc44ce3b80bddb10828dee3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:13,311 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:13,312 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=13, startTime=1732504512883; duration=0sec 2024-11-25T03:15:13,312 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:13,312 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:13,312 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:13,314 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:13,314 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:13,314 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:13,314 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f71e084b004f4d5291131dc56e7f60d1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/269d7706a78642d2afacd03daba92bac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6fd2b09b2e094cf0998c420c1efc5d20] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.6 K 2024-11-25T03:15:13,314 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f71e084b004f4d5291131dc56e7f60d1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732504508701 2024-11-25T03:15:13,315 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 269d7706a78642d2afacd03daba92bac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732504509851 2024-11-25T03:15:13,315 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fd2b09b2e094cf0998c420c1efc5d20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732504512019 2024-11-25T03:15:13,315 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into f2e28e090ff64d85b9d1827b68edd0c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:13,315 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:13,315 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=13, startTime=1732504512883; duration=0sec 2024-11-25T03:15:13,315 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:13,315 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:13,316 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/f3aad8893c104560b770456a502668b6 2024-11-25T03:15:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8a1cc5433b814a9786f8f1eb620aba86 is 50, key is test_row_0/B:col10/1732504512090/Put/seqid=0 2024-11-25T03:15:13,327 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:13,328 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/89d699c9b06343768e676f2bad7252bb is 50, key is test_row_0/C:col10/1732504512019/Put/seqid=0 2024-11-25T03:15:13,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742133_1309 (size=12301) 2024-11-25T03:15:13,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742134_1310 (size=12983) 2024-11-25T03:15:13,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504573369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504573370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504573370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-25T03:15:13,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504573579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504573580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504573580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,738 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8a1cc5433b814a9786f8f1eb620aba86 2024-11-25T03:15:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d24fee82bc664327a232904fd4d0bc2c is 50, key is test_row_0/C:col10/1732504512090/Put/seqid=0 2024-11-25T03:15:13,749 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/89d699c9b06343768e676f2bad7252bb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/89d699c9b06343768e676f2bad7252bb 2024-11-25T03:15:13,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742135_1311 (size=12301) 2024-11-25T03:15:13,755 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into 89d699c9b06343768e676f2bad7252bb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:13,755 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:13,755 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=13, startTime=1732504512883; duration=0sec 2024-11-25T03:15:13,755 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:13,755 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504573890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504573890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504573890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,153 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d24fee82bc664327a232904fd4d0bc2c 2024-11-25T03:15:14,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/f3aad8893c104560b770456a502668b6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/f3aad8893c104560b770456a502668b6 2024-11-25T03:15:14,162 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/f3aad8893c104560b770456a502668b6, entries=150, sequenceid=319, filesize=12.0 K 2024-11-25T03:15:14,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/8a1cc5433b814a9786f8f1eb620aba86 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8a1cc5433b814a9786f8f1eb620aba86 2024-11-25T03:15:14,166 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8a1cc5433b814a9786f8f1eb620aba86, entries=150, sequenceid=319, filesize=12.0 K 2024-11-25T03:15:14,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d24fee82bc664327a232904fd4d0bc2c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d24fee82bc664327a232904fd4d0bc2c 2024-11-25T03:15:14,175 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d24fee82bc664327a232904fd4d0bc2c, entries=150, sequenceid=319, filesize=12.0 K 2024-11-25T03:15:14,176 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 897f6457507f38ffcd25c9bdd6308219 in 1269ms, sequenceid=319, compaction requested=false 2024-11-25T03:15:14,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:14,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-25T03:15:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-25T03:15:14,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-25T03:15:14,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-25T03:15:14,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.8890 sec 2024-11-25T03:15:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:14,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T03:15:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:14,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/d2fe66fe857c4048b9a78bfe87eb37e3 is 50, key is test_row_0/A:col10/1732504513254/Put/seqid=0 2024-11-25T03:15:14,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742136_1312 (size=17181) 2024-11-25T03:15:14,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/d2fe66fe857c4048b9a78bfe87eb37e3 2024-11-25T03:15:14,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/df18c80531074ed891636f2c088691b2 is 50, key is test_row_0/B:col10/1732504513254/Put/seqid=0 2024-11-25T03:15:14,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742137_1313 (size=12301) 2024-11-25T03:15:14,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504574429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504574430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504574432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-25T03:15:14,440 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:14,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-25T03:15:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T03:15:14,442 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:14,442 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:14,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504574540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504574540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504574540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T03:15:14,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-25T03:15:14,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:14,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:14,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:14,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-25T03:15:14,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:14,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:14,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504574750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504574750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504574750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T03:15:14,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/df18c80531074ed891636f2c088691b2 2024-11-25T03:15:14,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/91a30a3d744d40258dbbb8fa5d1d643b is 50, key is test_row_0/C:col10/1732504513254/Put/seqid=0 2024-11-25T03:15:14,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742138_1314 (size=12301) 2024-11-25T03:15:14,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/91a30a3d744d40258dbbb8fa5d1d643b 2024-11-25T03:15:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/d2fe66fe857c4048b9a78bfe87eb37e3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d2fe66fe857c4048b9a78bfe87eb37e3 2024-11-25T03:15:14,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d2fe66fe857c4048b9a78bfe87eb37e3, entries=250, sequenceid=336, filesize=16.8 K 2024-11-25T03:15:14,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/df18c80531074ed891636f2c088691b2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/df18c80531074ed891636f2c088691b2 2024-11-25T03:15:14,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/df18c80531074ed891636f2c088691b2, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T03:15:14,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/91a30a3d744d40258dbbb8fa5d1d643b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/91a30a3d744d40258dbbb8fa5d1d643b 2024-11-25T03:15:14,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/91a30a3d744d40258dbbb8fa5d1d643b, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T03:15:14,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 897f6457507f38ffcd25c9bdd6308219 in 489ms, sequenceid=336, compaction requested=true 2024-11-25T03:15:14,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:14,893 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:14,893 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:14,894 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:14,894 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:14,894 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,894 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2d6b654aafc44ce3b80bddb10828dee3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/f3aad8893c104560b770456a502668b6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d2fe66fe857c4048b9a78bfe87eb37e3] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=41.5 K 2024-11-25T03:15:14,896 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:14,896 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:14,896 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,896 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6b654aafc44ce3b80bddb10828dee3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732504512019 2024-11-25T03:15:14,896 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/f2e28e090ff64d85b9d1827b68edd0c2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8a1cc5433b814a9786f8f1eb620aba86, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/df18c80531074ed891636f2c088691b2] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.7 K 2024-11-25T03:15:14,897 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3aad8893c104560b770456a502668b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732504512079 2024-11-25T03:15:14,897 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e28e090ff64d85b9d1827b68edd0c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732504512019 2024-11-25T03:15:14,897 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2fe66fe857c4048b9a78bfe87eb37e3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732504513250 2024-11-25T03:15:14,897 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a1cc5433b814a9786f8f1eb620aba86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732504512079 2024-11-25T03:15:14,898 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting df18c80531074ed891636f2c088691b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732504513254 2024-11-25T03:15:14,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-25T03:15:14,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:14,902 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:15:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:14,917 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:14,917 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/ced9761a48a5415ca611d4de70701aa3 is 50, key is test_row_0/A:col10/1732504513254/Put/seqid=0 2024-11-25T03:15:14,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/614e2f75a88244cd92482bb74c2b10b6 is 50, key is test_row_0/A:col10/1732504514428/Put/seqid=0 2024-11-25T03:15:14,927 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:14,927 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/ede72fc12609489c86aea2abb6ac6362 is 50, key is test_row_0/B:col10/1732504513254/Put/seqid=0 2024-11-25T03:15:14,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742140_1316 (size=12301) 2024-11-25T03:15:14,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742139_1315 (size=13085) 2024-11-25T03:15:14,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742141_1317 (size=13085) 2024-11-25T03:15:15,002 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/ede72fc12609489c86aea2abb6ac6362 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ede72fc12609489c86aea2abb6ac6362 2024-11-25T03:15:15,013 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into ede72fc12609489c86aea2abb6ac6362(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:15,013 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:15,013 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=13, startTime=1732504514893; duration=0sec 2024-11-25T03:15:15,013 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:15,013 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:15,013 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:15,018 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:15,018 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:15,019 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:15,019 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/89d699c9b06343768e676f2bad7252bb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d24fee82bc664327a232904fd4d0bc2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/91a30a3d744d40258dbbb8fa5d1d643b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.7 K 2024-11-25T03:15:15,020 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d699c9b06343768e676f2bad7252bb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732504512019 2024-11-25T03:15:15,022 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d24fee82bc664327a232904fd4d0bc2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732504512079 2024-11-25T03:15:15,023 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a30a3d744d40258dbbb8fa5d1d643b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732504513254 2024-11-25T03:15:15,044 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:15,044 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/57fc732a64d84a48b4723c0a90d1c24f is 50, key is test_row_0/C:col10/1732504513254/Put/seqid=0 2024-11-25T03:15:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:15,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742142_1318 (size=13085) 2024-11-25T03:15:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T03:15:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504575091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504575091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504575091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504575200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504575200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504575200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,379 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/614e2f75a88244cd92482bb74c2b10b6 2024-11-25T03:15:15,386 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/ced9761a48a5415ca611d4de70701aa3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/ced9761a48a5415ca611d4de70701aa3 2024-11-25T03:15:15,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/271d077442674fedbd2c496a4106b9ee is 50, key is test_row_0/B:col10/1732504514428/Put/seqid=0 2024-11-25T03:15:15,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742143_1319 (size=12301) 2024-11-25T03:15:15,398 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into ced9761a48a5415ca611d4de70701aa3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:15,398 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:15,398 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=13, startTime=1732504514893; duration=0sec 2024-11-25T03:15:15,398 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:15,398 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:15,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504575410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504575410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504575410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,475 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/57fc732a64d84a48b4723c0a90d1c24f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/57fc732a64d84a48b4723c0a90d1c24f 2024-11-25T03:15:15,479 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into 57fc732a64d84a48b4723c0a90d1c24f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:15,479 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:15,479 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=13, startTime=1732504514893; duration=0sec 2024-11-25T03:15:15,479 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:15,479 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T03:15:15,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504575720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504575720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504575721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:15,798 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/271d077442674fedbd2c496a4106b9ee 2024-11-25T03:15:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/b892924891c94359a7c1b2713810083b is 50, key is test_row_0/C:col10/1732504514428/Put/seqid=0 2024-11-25T03:15:15,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742144_1320 (size=12301) 2024-11-25T03:15:15,819 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/b892924891c94359a7c1b2713810083b 2024-11-25T03:15:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/614e2f75a88244cd92482bb74c2b10b6 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/614e2f75a88244cd92482bb74c2b10b6 2024-11-25T03:15:15,836 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/614e2f75a88244cd92482bb74c2b10b6, entries=150, sequenceid=354, filesize=12.0 K 2024-11-25T03:15:15,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/271d077442674fedbd2c496a4106b9ee as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/271d077442674fedbd2c496a4106b9ee 2024-11-25T03:15:15,842 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/271d077442674fedbd2c496a4106b9ee, entries=150, sequenceid=354, filesize=12.0 K 2024-11-25T03:15:15,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/b892924891c94359a7c1b2713810083b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/b892924891c94359a7c1b2713810083b 2024-11-25T03:15:15,851 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/b892924891c94359a7c1b2713810083b, entries=150, sequenceid=354, filesize=12.0 K 2024-11-25T03:15:15,853 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 897f6457507f38ffcd25c9bdd6308219 in 951ms, sequenceid=354, compaction requested=false 2024-11-25T03:15:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-25T03:15:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-25T03:15:15,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-25T03:15:15,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-11-25T03:15:15,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.4190 sec 2024-11-25T03:15:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:16,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T03:15:16,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:16,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:16,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:16,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:16,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:16,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:16,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/59a10a6162024d89ab628ab3b24c209a is 50, key is test_row_0/A:col10/1732504516233/Put/seqid=0 2024-11-25T03:15:16,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742145_1321 (size=14741) 2024-11-25T03:15:16,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/59a10a6162024d89ab628ab3b24c209a 2024-11-25T03:15:16,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/24fcc0a89f534eb28805d9d9ecb69552 is 50, key is test_row_0/B:col10/1732504516233/Put/seqid=0 2024-11-25T03:15:16,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742146_1322 (size=12301) 2024-11-25T03:15:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504576265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504576266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504576266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504576380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504576380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504576380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T03:15:16,590 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:16,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-25T03:15:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-25T03:15:16,592 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:16,593 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:16,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504576590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504576590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504576590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/24fcc0a89f534eb28805d9d9ecb69552 2024-11-25T03:15:16,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/4209745324b642d697f553c40045d172 is 50, key is test_row_0/C:col10/1732504516233/Put/seqid=0 2024-11-25T03:15:16,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742147_1323 (size=12301) 2024-11-25T03:15:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-25T03:15:16,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-25T03:15:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:16,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:16,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:16,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59384 deadline: 1732504576750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,752 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59320 deadline: 1732504576800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,804 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 897f6457507f38ffcd25c9bdd6308219 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18317 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-25T03:15:16,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:16,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59370 deadline: 1732504576900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59308 deadline: 1732504576900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59394 deadline: 1732504576900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:15:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-25T03:15:17,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-25T03:15:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:17,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:17,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/4209745324b642d697f553c40045d172 2024-11-25T03:15:17,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/59a10a6162024d89ab628ab3b24c209a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/59a10a6162024d89ab628ab3b24c209a 2024-11-25T03:15:17,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/59a10a6162024d89ab628ab3b24c209a, entries=200, sequenceid=374, filesize=14.4 K 2024-11-25T03:15:17,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/24fcc0a89f534eb28805d9d9ecb69552 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/24fcc0a89f534eb28805d9d9ecb69552 2024-11-25T03:15:17,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/24fcc0a89f534eb28805d9d9ecb69552, entries=150, sequenceid=374, filesize=12.0 K 2024-11-25T03:15:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/4209745324b642d697f553c40045d172 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4209745324b642d697f553c40045d172 2024-11-25T03:15:17,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4209745324b642d697f553c40045d172, entries=150, sequenceid=374, filesize=12.0 K 2024-11-25T03:15:17,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 897f6457507f38ffcd25c9bdd6308219 in 861ms, sequenceid=374, compaction requested=true 2024-11-25T03:15:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:17,094 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 897f6457507f38ffcd25c9bdd6308219:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:17,094 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:17,096 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:17,096 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:17,096 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/B is initiating minor compaction (all files) 2024-11-25T03:15:17,096 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/A is initiating minor compaction (all files) 2024-11-25T03:15:17,096 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/B in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:17,096 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/A in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:17,097 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ede72fc12609489c86aea2abb6ac6362, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/271d077442674fedbd2c496a4106b9ee, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/24fcc0a89f534eb28805d9d9ecb69552] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.8 K 2024-11-25T03:15:17,097 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/ced9761a48a5415ca611d4de70701aa3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/614e2f75a88244cd92482bb74c2b10b6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/59a10a6162024d89ab628ab3b24c209a] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=39.2 K 2024-11-25T03:15:17,098 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ede72fc12609489c86aea2abb6ac6362, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732504513254 2024-11-25T03:15:17,098 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ced9761a48a5415ca611d4de70701aa3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732504513254 2024-11-25T03:15:17,098 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 614e2f75a88244cd92482bb74c2b10b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732504514426 2024-11-25T03:15:17,098 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 271d077442674fedbd2c496a4106b9ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732504514426 2024-11-25T03:15:17,099 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a10a6162024d89ab628ab3b24c209a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732504515078 2024-11-25T03:15:17,099 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 24fcc0a89f534eb28805d9d9ecb69552, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732504515087 2024-11-25T03:15:17,106 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:17,107 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/06210dbb987744c2b20d841c639f0e28 is 50, key is test_row_0/B:col10/1732504516233/Put/seqid=0 2024-11-25T03:15:17,109 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:17,110 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/b48e8fe7bc0a4d9c9d1c464f48b3cfdb is 50, key is test_row_0/A:col10/1732504516233/Put/seqid=0 2024-11-25T03:15:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742148_1324 (size=13187) 2024-11-25T03:15:17,119 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/06210dbb987744c2b20d841c639f0e28 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/06210dbb987744c2b20d841c639f0e28 2024-11-25T03:15:17,124 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-25T03:15:17,124 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,124 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,124 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,124 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-25T03:15:17,124 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,124 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,124 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,125 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/B of 897f6457507f38ffcd25c9bdd6308219 into 06210dbb987744c2b20d841c639f0e28(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:17,125 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:17,125 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/B, priority=13, startTime=1732504517094; duration=0sec 2024-11-25T03:15:17,125 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:17,125 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:B 2024-11-25T03:15:17,125 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:17,126 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-25T03:15:17,126 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,126 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,126 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,127 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-25T03:15:17,127 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,127 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:17,127 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,127 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 897f6457507f38ffcd25c9bdd6308219/C is initiating minor compaction (all files) 2024-11-25T03:15:17,127 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,127 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-25T03:15:17,127 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 897f6457507f38ffcd25c9bdd6308219/C in TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:17,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,127 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,127 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,127 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,127 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/57fc732a64d84a48b4723c0a90d1c24f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/b892924891c94359a7c1b2713810083b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4209745324b642d697f553c40045d172] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp, totalSize=36.8 K 2024-11-25T03:15:17,128 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 57fc732a64d84a48b4723c0a90d1c24f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732504513254 2024-11-25T03:15:17,128 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b892924891c94359a7c1b2713810083b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732504514426 2024-11-25T03:15:17,129 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4209745324b642d697f553c40045d172, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732504515087 2024-11-25T03:15:17,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742149_1325 (size=13187) 2024-11-25T03:15:17,134 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/b48e8fe7bc0a4d9c9d1c464f48b3cfdb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b48e8fe7bc0a4d9c9d1c464f48b3cfdb 2024-11-25T03:15:17,136 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 897f6457507f38ffcd25c9bdd6308219#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:17,137 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d182e330f42846a1b931b5965cfc46de is 50, key is test_row_0/C:col10/1732504516233/Put/seqid=0 2024-11-25T03:15:17,138 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/A of 897f6457507f38ffcd25c9bdd6308219 into b48e8fe7bc0a4d9c9d1c464f48b3cfdb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:17,138 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:17,138 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/A, priority=13, startTime=1732504517094; duration=0sec 2024-11-25T03:15:17,138 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:17,138 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:A 2024-11-25T03:15:17,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742150_1326 (size=13187) 2024-11-25T03:15:17,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-25T03:15:17,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:17,207 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:15:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6a98882d1cb04d04a7a5c543aa97114a is 50, key is test_row_0/A:col10/1732504516262/Put/seqid=0 2024-11-25T03:15:17,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742151_1327 (size=12301) 2024-11-25T03:15:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-25T03:15:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:17,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. as already flushing 2024-11-25T03:15:17,412 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-25T03:15:17,412 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-25T03:15:17,412 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-25T03:15:17,412 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,412 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,412 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:17,412 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,412 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,412 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,412 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,412 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,412 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:17,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:17,544 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/d182e330f42846a1b931b5965cfc46de as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d182e330f42846a1b931b5965cfc46de 2024-11-25T03:15:17,548 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 897f6457507f38ffcd25c9bdd6308219/C of 897f6457507f38ffcd25c9bdd6308219 into d182e330f42846a1b931b5965cfc46de(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:17,548 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:17,548 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., storeName=897f6457507f38ffcd25c9bdd6308219/C, priority=13, startTime=1732504517094; duration=0sec 2024-11-25T03:15:17,548 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:17,548 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 897f6457507f38ffcd25c9bdd6308219:C 2024-11-25T03:15:17,615 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6a98882d1cb04d04a7a5c543aa97114a 2024-11-25T03:15:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/0760599be6ff4c689a22b7449d4a802b is 50, key is test_row_0/B:col10/1732504516262/Put/seqid=0 2024-11-25T03:15:17,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742152_1328 (size=12301) 2024-11-25T03:15:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-25T03:15:18,025 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/0760599be6ff4c689a22b7449d4a802b 2024-11-25T03:15:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/ae53157e07754ff4a69fac39e33e8a58 is 50, key is test_row_0/C:col10/1732504516262/Put/seqid=0 2024-11-25T03:15:18,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742153_1329 (size=12301) 2024-11-25T03:15:18,435 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/ae53157e07754ff4a69fac39e33e8a58 2024-11-25T03:15:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/6a98882d1cb04d04a7a5c543aa97114a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6a98882d1cb04d04a7a5c543aa97114a 2024-11-25T03:15:18,441 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6a98882d1cb04d04a7a5c543aa97114a, entries=150, sequenceid=393, filesize=12.0 K 2024-11-25T03:15:18,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/0760599be6ff4c689a22b7449d4a802b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/0760599be6ff4c689a22b7449d4a802b 2024-11-25T03:15:18,445 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/0760599be6ff4c689a22b7449d4a802b, entries=150, sequenceid=393, filesize=12.0 K 2024-11-25T03:15:18,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/ae53157e07754ff4a69fac39e33e8a58 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/ae53157e07754ff4a69fac39e33e8a58 2024-11-25T03:15:18,448 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/ae53157e07754ff4a69fac39e33e8a58, entries=150, sequenceid=393, filesize=12.0 K 2024-11-25T03:15:18,449 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 897f6457507f38ffcd25c9bdd6308219 in 1243ms, sequenceid=393, compaction requested=false 2024-11-25T03:15:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 897f6457507f38ffcd25c9bdd6308219: 2024-11-25T03:15:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-25T03:15:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-25T03:15:18,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-25T03:15:18,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-11-25T03:15:18,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.8610 sec 2024-11-25T03:15:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-25T03:15:18,741 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:26,822 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-25T03:15:26,822 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:26,822 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:26,822 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:26,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:26,881 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-25T03:15:26,881 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:26,881 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:26,881 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:26,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3337 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3092 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3033 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3372 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3089 2024-11-25T03:15:26,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:15:26,882 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:15:26,882 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:15:26,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:26,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:26,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:26,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:15:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-25T03:15:26,886 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504526885"}]},"ts":"1732504526885"} 2024-11-25T03:15:26,887 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:15:26,887 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:15:26,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:15:26,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, UNASSIGN}] 2024-11-25T03:15:26,889 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, UNASSIGN 2024-11-25T03:15:26,890 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=897f6457507f38ffcd25c9bdd6308219, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:26,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, UNASSIGN because future has completed 2024-11-25T03:15:26,891 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:15:26,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:15:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-25T03:15:27,043 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:27,043 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:15:27,043 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 897f6457507f38ffcd25c9bdd6308219, disabling compactions & flushes 2024-11-25T03:15:27,043 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:27,043 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:27,043 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. after waiting 0 ms 2024-11-25T03:15:27,043 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:27,044 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing 897f6457507f38ffcd25c9bdd6308219 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T03:15:27,044 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=A 2024-11-25T03:15:27,044 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:27,044 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=B 2024-11-25T03:15:27,044 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:27,044 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 897f6457507f38ffcd25c9bdd6308219, store=C 2024-11-25T03:15:27,044 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:27,047 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2b2a7bc7d82e4748be7f814c6442dd92 is 50, key is test_row_0/A:col10/1732504517411/Put/seqid=0 2024-11-25T03:15:27,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742154_1330 (size=12301) 2024-11-25T03:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-25T03:15:27,451 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2b2a7bc7d82e4748be7f814c6442dd92 2024-11-25T03:15:27,457 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/08cb1a5891e44dd6a6a06dbeab7bab96 is 50, key is test_row_0/B:col10/1732504517411/Put/seqid=0 2024-11-25T03:15:27,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742155_1331 (size=12301) 2024-11-25T03:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-25T03:15:27,861 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/08cb1a5891e44dd6a6a06dbeab7bab96 2024-11-25T03:15:27,867 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/dad88ed9d8974e4e85ca53c70673c2a7 is 50, key is test_row_0/C:col10/1732504517411/Put/seqid=0 2024-11-25T03:15:27,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742156_1332 (size=12301) 2024-11-25T03:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-25T03:15:28,271 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/dad88ed9d8974e4e85ca53c70673c2a7 2024-11-25T03:15:28,275 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/A/2b2a7bc7d82e4748be7f814c6442dd92 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2b2a7bc7d82e4748be7f814c6442dd92 2024-11-25T03:15:28,278 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2b2a7bc7d82e4748be7f814c6442dd92, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T03:15:28,279 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/B/08cb1a5891e44dd6a6a06dbeab7bab96 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/08cb1a5891e44dd6a6a06dbeab7bab96 2024-11-25T03:15:28,282 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/08cb1a5891e44dd6a6a06dbeab7bab96, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T03:15:28,282 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/.tmp/C/dad88ed9d8974e4e85ca53c70673c2a7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/dad88ed9d8974e4e85ca53c70673c2a7 2024-11-25T03:15:28,285 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/dad88ed9d8974e4e85ca53c70673c2a7, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T03:15:28,286 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 897f6457507f38ffcd25c9bdd6308219 in 1243ms, sequenceid=402, compaction requested=true 2024-11-25T03:15:28,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/106700eb075d4a7c909afe92fdd0fed2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e17c9813e8c44490be153fd3e176e7f7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/8302199595204df28e31f94722b802c7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2a4e56c548574b708b0a5e902e284468, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/90ecfdbc9ff549688ed5b3a3312570d8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/69684f4386b5464287561d99a67403f1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/bf1830dea3184973aa2470fa1b51f394, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6425bd32bfbf403eb425783ec8c6ba0e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e0f4d48ad81e4d4f9d34cba643e7cfc3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/12259611e64148dc93136a9d31e479cd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b5b793af0a294852a2a1da91b67a3a4e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/acde60b20b5242e98c36ff9bdd1fd0cd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10f9c5ca4ce446f392ec38756db0395e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/fce55a26eb01413b842736df0ffedd8e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/62c26fcccccf4e0fa0df3903c9d21193, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6eb71d0e53f3476ca6bcf6aad3d212bc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6efd1131b4c442b0b67707c1501c4396, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6db3201742574a5c96d83a890225b24b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d702d098ebcc4ec580f6512d55226d4e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10b53ba3a7d14dca96514be6b6714bcc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2d6b654aafc44ce3b80bddb10828dee3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/f3aad8893c104560b770456a502668b6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d2fe66fe857c4048b9a78bfe87eb37e3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/ced9761a48a5415ca611d4de70701aa3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/614e2f75a88244cd92482bb74c2b10b6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/59a10a6162024d89ab628ab3b24c209a] to archive 2024-11-25T03:15:28,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:28,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/106700eb075d4a7c909afe92fdd0fed2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/106700eb075d4a7c909afe92fdd0fed2 2024-11-25T03:15:28,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e17c9813e8c44490be153fd3e176e7f7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e17c9813e8c44490be153fd3e176e7f7 2024-11-25T03:15:28,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/8302199595204df28e31f94722b802c7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/8302199595204df28e31f94722b802c7 2024-11-25T03:15:28,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2a4e56c548574b708b0a5e902e284468 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2a4e56c548574b708b0a5e902e284468 2024-11-25T03:15:28,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/90ecfdbc9ff549688ed5b3a3312570d8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/90ecfdbc9ff549688ed5b3a3312570d8 2024-11-25T03:15:28,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/69684f4386b5464287561d99a67403f1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/69684f4386b5464287561d99a67403f1 2024-11-25T03:15:28,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/bf1830dea3184973aa2470fa1b51f394 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/bf1830dea3184973aa2470fa1b51f394 2024-11-25T03:15:28,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6425bd32bfbf403eb425783ec8c6ba0e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6425bd32bfbf403eb425783ec8c6ba0e 2024-11-25T03:15:28,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e0f4d48ad81e4d4f9d34cba643e7cfc3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/e0f4d48ad81e4d4f9d34cba643e7cfc3 2024-11-25T03:15:28,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/12259611e64148dc93136a9d31e479cd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/12259611e64148dc93136a9d31e479cd 2024-11-25T03:15:28,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b5b793af0a294852a2a1da91b67a3a4e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b5b793af0a294852a2a1da91b67a3a4e 2024-11-25T03:15:28,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/acde60b20b5242e98c36ff9bdd1fd0cd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/acde60b20b5242e98c36ff9bdd1fd0cd 2024-11-25T03:15:28,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10f9c5ca4ce446f392ec38756db0395e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10f9c5ca4ce446f392ec38756db0395e 2024-11-25T03:15:28,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/fce55a26eb01413b842736df0ffedd8e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/fce55a26eb01413b842736df0ffedd8e 2024-11-25T03:15:28,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/62c26fcccccf4e0fa0df3903c9d21193 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/62c26fcccccf4e0fa0df3903c9d21193 2024-11-25T03:15:28,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6eb71d0e53f3476ca6bcf6aad3d212bc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6eb71d0e53f3476ca6bcf6aad3d212bc 2024-11-25T03:15:28,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6efd1131b4c442b0b67707c1501c4396 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6efd1131b4c442b0b67707c1501c4396 2024-11-25T03:15:28,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6db3201742574a5c96d83a890225b24b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6db3201742574a5c96d83a890225b24b 2024-11-25T03:15:28,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d702d098ebcc4ec580f6512d55226d4e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d702d098ebcc4ec580f6512d55226d4e 2024-11-25T03:15:28,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10b53ba3a7d14dca96514be6b6714bcc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/10b53ba3a7d14dca96514be6b6714bcc 2024-11-25T03:15:28,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2d6b654aafc44ce3b80bddb10828dee3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2d6b654aafc44ce3b80bddb10828dee3 2024-11-25T03:15:28,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/f3aad8893c104560b770456a502668b6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/f3aad8893c104560b770456a502668b6 2024-11-25T03:15:28,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d2fe66fe857c4048b9a78bfe87eb37e3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/d2fe66fe857c4048b9a78bfe87eb37e3 2024-11-25T03:15:28,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/ced9761a48a5415ca611d4de70701aa3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/ced9761a48a5415ca611d4de70701aa3 2024-11-25T03:15:28,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/614e2f75a88244cd92482bb74c2b10b6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/614e2f75a88244cd92482bb74c2b10b6 2024-11-25T03:15:28,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/59a10a6162024d89ab628ab3b24c209a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/59a10a6162024d89ab628ab3b24c209a 2024-11-25T03:15:28,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8f40fd47d4cb42768a5f6d3652efba83, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/866b6e4d49f74a339fc511acd9939198, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9c3878075f544fd0a7712a73ad3b485e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/de94e54d54b540a7ade89ebc54aa2e92, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/b704b35c30e64db8997ca168b0caa0ea, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/20734d4ea33e479db352fea34e9c2ba1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/db5a357de98d4b13a8dfb60a4b65f245, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/d93a0f7705914255b7e24fda4afd830c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/bc615e3af22e4af08fc491b4dc9a601d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ebd53a7b43ee4ba4a5b66113746de762, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9b3276aecd0044aab2e38172308ac7e4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/c04bb7173ceb4b3e94b623a21e8f31a1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8dafcb074a59495aa9ac61cfac653391, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/05efced1be564b55b9de195b14e98115, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/79b3b6e50fc941458a1c9b42d6b8bd9d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/7ddf083b881546c5b84d7f517134d814, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/e727d2078c3e4360ba69a4e7bd5287c5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/74f09574c5a841878bc02cb9845c7fbb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/4a49c70769ab49f6a86df0c6f29bdd45, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/f2e28e090ff64d85b9d1827b68edd0c2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/3efdae32647047efb3a3f71dc9b452a6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8a1cc5433b814a9786f8f1eb620aba86, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ede72fc12609489c86aea2abb6ac6362, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/df18c80531074ed891636f2c088691b2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/271d077442674fedbd2c496a4106b9ee, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/24fcc0a89f534eb28805d9d9ecb69552] to archive 2024-11-25T03:15:28,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:28,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8f40fd47d4cb42768a5f6d3652efba83 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8f40fd47d4cb42768a5f6d3652efba83 2024-11-25T03:15:28,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/866b6e4d49f74a339fc511acd9939198 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/866b6e4d49f74a339fc511acd9939198 2024-11-25T03:15:28,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9c3878075f544fd0a7712a73ad3b485e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9c3878075f544fd0a7712a73ad3b485e 2024-11-25T03:15:28,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/de94e54d54b540a7ade89ebc54aa2e92 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/de94e54d54b540a7ade89ebc54aa2e92 2024-11-25T03:15:28,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/b704b35c30e64db8997ca168b0caa0ea to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/b704b35c30e64db8997ca168b0caa0ea 2024-11-25T03:15:28,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/20734d4ea33e479db352fea34e9c2ba1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/20734d4ea33e479db352fea34e9c2ba1 2024-11-25T03:15:28,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/db5a357de98d4b13a8dfb60a4b65f245 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/db5a357de98d4b13a8dfb60a4b65f245 2024-11-25T03:15:28,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/d93a0f7705914255b7e24fda4afd830c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/d93a0f7705914255b7e24fda4afd830c 2024-11-25T03:15:28,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/bc615e3af22e4af08fc491b4dc9a601d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/bc615e3af22e4af08fc491b4dc9a601d 2024-11-25T03:15:28,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ebd53a7b43ee4ba4a5b66113746de762 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ebd53a7b43ee4ba4a5b66113746de762 2024-11-25T03:15:28,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9b3276aecd0044aab2e38172308ac7e4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/9b3276aecd0044aab2e38172308ac7e4 2024-11-25T03:15:28,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/c04bb7173ceb4b3e94b623a21e8f31a1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/c04bb7173ceb4b3e94b623a21e8f31a1 2024-11-25T03:15:28,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8dafcb074a59495aa9ac61cfac653391 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8dafcb074a59495aa9ac61cfac653391 2024-11-25T03:15:28,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/05efced1be564b55b9de195b14e98115 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/05efced1be564b55b9de195b14e98115 2024-11-25T03:15:28,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/79b3b6e50fc941458a1c9b42d6b8bd9d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/79b3b6e50fc941458a1c9b42d6b8bd9d 2024-11-25T03:15:28,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/7ddf083b881546c5b84d7f517134d814 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/7ddf083b881546c5b84d7f517134d814 2024-11-25T03:15:28,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/e727d2078c3e4360ba69a4e7bd5287c5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/e727d2078c3e4360ba69a4e7bd5287c5 2024-11-25T03:15:28,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/74f09574c5a841878bc02cb9845c7fbb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/74f09574c5a841878bc02cb9845c7fbb 2024-11-25T03:15:28,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/4a49c70769ab49f6a86df0c6f29bdd45 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/4a49c70769ab49f6a86df0c6f29bdd45 2024-11-25T03:15:28,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/f2e28e090ff64d85b9d1827b68edd0c2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/f2e28e090ff64d85b9d1827b68edd0c2 2024-11-25T03:15:28,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/3efdae32647047efb3a3f71dc9b452a6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/3efdae32647047efb3a3f71dc9b452a6 2024-11-25T03:15:28,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8a1cc5433b814a9786f8f1eb620aba86 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/8a1cc5433b814a9786f8f1eb620aba86 2024-11-25T03:15:28,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ede72fc12609489c86aea2abb6ac6362 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/ede72fc12609489c86aea2abb6ac6362 2024-11-25T03:15:28,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/df18c80531074ed891636f2c088691b2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/df18c80531074ed891636f2c088691b2 2024-11-25T03:15:28,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/271d077442674fedbd2c496a4106b9ee to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/271d077442674fedbd2c496a4106b9ee 2024-11-25T03:15:28,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/24fcc0a89f534eb28805d9d9ecb69552 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/24fcc0a89f534eb28805d9d9ecb69552 2024-11-25T03:15:28,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/0821195674f1491da959f2cef31102d1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4939e79d0f9e43979440a4964377b2c8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/2cf4c6ebfc784b179c38b6f83a9a8340, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d9a4b5c11d824899a92f1d116a9dc911, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/83473ca15f314b3a912630f86d54cc13, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/cc7799c4fa0b49cc9424db53c6a3be22, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3a5b65607fbd4efbb642fdee22bd501d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/52ad47599676416dae1ad2c50cf86aa1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3ef8b21e827840cc83f579e169a3cc5a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3d781f060e38497cbdd25ed6f710bb2a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/61647c0c9ee54521baa47aa7f5d109e4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/919b10baa3c248158f52c8fbb4e483a2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6e362c70d41c4cabb10d903d01986319, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/27a2ac33d6384edb9d430aacebf1acc6, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f5fb2f6ed3884702ac0cb736ced13d8f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/5ece1447db554a369d20c2f8dc4c16ba, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f71e084b004f4d5291131dc56e7f60d1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/436f3324e09149bcbc7b10529f318cac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/269d7706a78642d2afacd03daba92bac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/89d699c9b06343768e676f2bad7252bb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6fd2b09b2e094cf0998c420c1efc5d20, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d24fee82bc664327a232904fd4d0bc2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/57fc732a64d84a48b4723c0a90d1c24f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/91a30a3d744d40258dbbb8fa5d1d643b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/b892924891c94359a7c1b2713810083b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4209745324b642d697f553c40045d172] to archive 2024-11-25T03:15:28,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:28,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/0821195674f1491da959f2cef31102d1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/0821195674f1491da959f2cef31102d1 2024-11-25T03:15:28,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4939e79d0f9e43979440a4964377b2c8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4939e79d0f9e43979440a4964377b2c8 2024-11-25T03:15:28,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/2cf4c6ebfc784b179c38b6f83a9a8340 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/2cf4c6ebfc784b179c38b6f83a9a8340 2024-11-25T03:15:28,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d9a4b5c11d824899a92f1d116a9dc911 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d9a4b5c11d824899a92f1d116a9dc911 2024-11-25T03:15:28,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/83473ca15f314b3a912630f86d54cc13 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/83473ca15f314b3a912630f86d54cc13 2024-11-25T03:15:28,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/cc7799c4fa0b49cc9424db53c6a3be22 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/cc7799c4fa0b49cc9424db53c6a3be22 2024-11-25T03:15:28,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3a5b65607fbd4efbb642fdee22bd501d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3a5b65607fbd4efbb642fdee22bd501d 2024-11-25T03:15:28,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/52ad47599676416dae1ad2c50cf86aa1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/52ad47599676416dae1ad2c50cf86aa1 2024-11-25T03:15:28,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3ef8b21e827840cc83f579e169a3cc5a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3ef8b21e827840cc83f579e169a3cc5a 2024-11-25T03:15:28,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3d781f060e38497cbdd25ed6f710bb2a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/3d781f060e38497cbdd25ed6f710bb2a 2024-11-25T03:15:28,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/61647c0c9ee54521baa47aa7f5d109e4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/61647c0c9ee54521baa47aa7f5d109e4 2024-11-25T03:15:28,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/919b10baa3c248158f52c8fbb4e483a2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/919b10baa3c248158f52c8fbb4e483a2 2024-11-25T03:15:28,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6e362c70d41c4cabb10d903d01986319 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6e362c70d41c4cabb10d903d01986319 2024-11-25T03:15:28,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/27a2ac33d6384edb9d430aacebf1acc6 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/27a2ac33d6384edb9d430aacebf1acc6 2024-11-25T03:15:28,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f5fb2f6ed3884702ac0cb736ced13d8f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f5fb2f6ed3884702ac0cb736ced13d8f 2024-11-25T03:15:28,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/5ece1447db554a369d20c2f8dc4c16ba to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/5ece1447db554a369d20c2f8dc4c16ba 2024-11-25T03:15:28,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f71e084b004f4d5291131dc56e7f60d1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/f71e084b004f4d5291131dc56e7f60d1 2024-11-25T03:15:28,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/436f3324e09149bcbc7b10529f318cac to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/436f3324e09149bcbc7b10529f318cac 2024-11-25T03:15:28,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/269d7706a78642d2afacd03daba92bac to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/269d7706a78642d2afacd03daba92bac 2024-11-25T03:15:28,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/89d699c9b06343768e676f2bad7252bb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/89d699c9b06343768e676f2bad7252bb 2024-11-25T03:15:28,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6fd2b09b2e094cf0998c420c1efc5d20 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/6fd2b09b2e094cf0998c420c1efc5d20 2024-11-25T03:15:28,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d24fee82bc664327a232904fd4d0bc2c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d24fee82bc664327a232904fd4d0bc2c 2024-11-25T03:15:28,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/57fc732a64d84a48b4723c0a90d1c24f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/57fc732a64d84a48b4723c0a90d1c24f 2024-11-25T03:15:28,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/91a30a3d744d40258dbbb8fa5d1d643b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/91a30a3d744d40258dbbb8fa5d1d643b 2024-11-25T03:15:28,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/b892924891c94359a7c1b2713810083b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/b892924891c94359a7c1b2713810083b 2024-11-25T03:15:28,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4209745324b642d697f553c40045d172 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/4209745324b642d697f553c40045d172 2024-11-25T03:15:28,361 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/recovered.edits/405.seqid, newMaxSeqId=405, maxSeqId=1 2024-11-25T03:15:28,361 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. 2024-11-25T03:15:28,362 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 897f6457507f38ffcd25c9bdd6308219: Waiting for close lock at 1732504527043Running coprocessor pre-close hooks at 1732504527043Disabling compacts and flushes for region at 1732504527043Disabling writes for close at 1732504527043Obtaining lock to block concurrent updates at 1732504527044 (+1 ms)Preparing flush snapshotting stores in 897f6457507f38ffcd25c9bdd6308219 at 1732504527044Finished memstore snapshotting TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732504527044Flushing stores of TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219. at 1732504527044Flushing 897f6457507f38ffcd25c9bdd6308219/A: creating writer at 1732504527045 (+1 ms)Flushing 897f6457507f38ffcd25c9bdd6308219/A: appending metadata at 1732504527047 (+2 ms)Flushing 897f6457507f38ffcd25c9bdd6308219/A: closing flushed file at 1732504527047Flushing 897f6457507f38ffcd25c9bdd6308219/B: creating writer at 1732504527455 (+408 ms)Flushing 897f6457507f38ffcd25c9bdd6308219/B: appending metadata at 1732504527457 (+2 ms)Flushing 897f6457507f38ffcd25c9bdd6308219/B: closing flushed file at 1732504527457Flushing 897f6457507f38ffcd25c9bdd6308219/C: creating writer at 1732504527864 (+407 ms)Flushing 897f6457507f38ffcd25c9bdd6308219/C: appending metadata at 1732504527867 (+3 ms)Flushing 897f6457507f38ffcd25c9bdd6308219/C: closing flushed file at 1732504527867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3384f3eb: reopening flushed file at 1732504528275 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e712fd9: reopening flushed file at 1732504528278 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b21ded7: reopening flushed file at 1732504528282 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 897f6457507f38ffcd25c9bdd6308219 in 1243ms, sequenceid=402, compaction requested=true at 1732504528286 (+4 ms)Writing region close event to WAL at 1732504528358 (+72 ms)Running coprocessor post-close hooks at 1732504528361 (+3 ms)Closed at 1732504528361 2024-11-25T03:15:28,363 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:28,364 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=897f6457507f38ffcd25c9bdd6308219, regionState=CLOSED 2024-11-25T03:15:28,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:15:28,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-25T03:15:28,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 897f6457507f38ffcd25c9bdd6308219, server=38a5d2077268,43357,1732504425108 in 1.4750 sec 2024-11-25T03:15:28,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-25T03:15:28,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=897f6457507f38ffcd25c9bdd6308219, UNASSIGN in 1.4790 sec 2024-11-25T03:15:28,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-25T03:15:28,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4810 sec 2024-11-25T03:15:28,371 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504528371"}]},"ts":"1732504528371"} 2024-11-25T03:15:28,372 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:15:28,372 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:15:28,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4890 sec 2024-11-25T03:15:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-25T03:15:29,031 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:29,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:15:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:29,033 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-25T03:15:29,033 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:29,036 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:29,038 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/recovered.edits] 2024-11-25T03:15:29,040 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2b2a7bc7d82e4748be7f814c6442dd92 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/2b2a7bc7d82e4748be7f814c6442dd92 2024-11-25T03:15:29,042 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6a98882d1cb04d04a7a5c543aa97114a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/6a98882d1cb04d04a7a5c543aa97114a 2024-11-25T03:15:29,043 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b48e8fe7bc0a4d9c9d1c464f48b3cfdb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/A/b48e8fe7bc0a4d9c9d1c464f48b3cfdb 2024-11-25T03:15:29,044 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/06210dbb987744c2b20d841c639f0e28 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/06210dbb987744c2b20d841c639f0e28 2024-11-25T03:15:29,045 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/0760599be6ff4c689a22b7449d4a802b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/0760599be6ff4c689a22b7449d4a802b 2024-11-25T03:15:29,046 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/08cb1a5891e44dd6a6a06dbeab7bab96 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/B/08cb1a5891e44dd6a6a06dbeab7bab96 2024-11-25T03:15:29,048 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/ae53157e07754ff4a69fac39e33e8a58 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/ae53157e07754ff4a69fac39e33e8a58 2024-11-25T03:15:29,049 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d182e330f42846a1b931b5965cfc46de to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/d182e330f42846a1b931b5965cfc46de 2024-11-25T03:15:29,050 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/dad88ed9d8974e4e85ca53c70673c2a7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/C/dad88ed9d8974e4e85ca53c70673c2a7 2024-11-25T03:15:29,053 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/recovered.edits/405.seqid to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219/recovered.edits/405.seqid 2024-11-25T03:15:29,053 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/897f6457507f38ffcd25c9bdd6308219 2024-11-25T03:15:29,053 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:15:29,055 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:29,058 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:15:29,059 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:15:29,060 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:29,060 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:15:29,060 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732504529060"}]},"ts":"9223372036854775807"} 2024-11-25T03:15:29,062 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:15:29,062 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 897f6457507f38ffcd25c9bdd6308219, NAME => 'TestAcidGuarantees,,1732504494919.897f6457507f38ffcd25c9bdd6308219.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:15:29,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:15:29,062 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732504529062"}]},"ts":"9223372036854775807"} 2024-11-25T03:15:29,064 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:15:29,064 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:29,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-25T03:15:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-25T03:15:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:15:29,141 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:29,150 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247 (was 248), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 457), ProcessCount=11 (was 11), AvailableMemoryMB=6469 (was 7082) 2024-11-25T03:15:29,158 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=6469 2024-11-25T03:15:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:15:29,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:15:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:29,162 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:15:29,162 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:29,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-25T03:15:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:15:29,163 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:15:29,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742157_1333 (size=1061) 2024-11-25T03:15:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:15:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:15:29,570 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:15:29,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742158_1334 (size=53) 2024-11-25T03:15:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:15:29,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:15:29,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bcf6a7b706584359872cc54bd64d673e, disabling compactions & flushes 2024-11-25T03:15:29,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:29,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:29,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. after waiting 0 ms 2024-11-25T03:15:29,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:29,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:29,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bcf6a7b706584359872cc54bd64d673e: Waiting for close lock at 1732504529975Disabling compacts and flushes for region at 1732504529975Disabling writes for close at 1732504529976 (+1 ms)Writing region close event to WAL at 1732504529976Closed at 1732504529976 2024-11-25T03:15:29,977 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:15:29,977 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732504529977"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732504529977"}]},"ts":"1732504529977"} 2024-11-25T03:15:29,978 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:15:29,979 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:15:29,979 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504529979"}]},"ts":"1732504529979"} 2024-11-25T03:15:29,980 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:15:29,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, ASSIGN}] 2024-11-25T03:15:29,981 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, ASSIGN 2024-11-25T03:15:29,982 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, ASSIGN; state=OFFLINE, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=false 2024-11-25T03:15:30,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:30,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, ASSIGN because future has completed 2024-11-25T03:15:30,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:15:30,289 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:30,289 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:15:30,289 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,289 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:15:30,289 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,289 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,290 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,291 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:30,292 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf6a7b706584359872cc54bd64d673e columnFamilyName A 2024-11-25T03:15:30,292 DEBUG [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:30,292 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(327): Store=bcf6a7b706584359872cc54bd64d673e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:30,292 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,293 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:30,293 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf6a7b706584359872cc54bd64d673e columnFamilyName B 2024-11-25T03:15:30,293 DEBUG [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:30,293 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(327): Store=bcf6a7b706584359872cc54bd64d673e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:30,293 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,294 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:30,294 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf6a7b706584359872cc54bd64d673e columnFamilyName C 2024-11-25T03:15:30,294 DEBUG [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:30,294 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(327): Store=bcf6a7b706584359872cc54bd64d673e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:30,295 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:30,295 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,295 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,295 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,296 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,296 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,297 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:15:30,298 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,299 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:15:30,300 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened bcf6a7b706584359872cc54bd64d673e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64076966, jitterRate=-0.04517880082130432}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:15:30,300 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:15:30,300 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for bcf6a7b706584359872cc54bd64d673e: Running coprocessor pre-open hook at 1732504530289Writing region info on filesystem at 1732504530289Initializing all the Stores at 1732504530290 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504530290Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504530290Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504530290Cleaning up temporary data from old regions at 1732504530296 (+6 ms)Running coprocessor post-open hooks at 1732504530300 (+4 ms)Region opened successfully at 1732504530300 2024-11-25T03:15:30,301 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., pid=99, masterSystemTime=1732504530286 2024-11-25T03:15:30,302 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:30,302 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:30,303 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:30,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:15:30,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-25T03:15:30,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 in 171 msec 2024-11-25T03:15:30,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-25T03:15:30,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, ASSIGN in 326 msec 2024-11-25T03:15:30,308 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:15:30,308 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504530308"}]},"ts":"1732504530308"} 2024-11-25T03:15:30,309 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:15:30,310 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:15:30,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-25T03:15:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:15:31,310 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:31,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:31,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:31,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:31,312 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51052c0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:31,314 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:31,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ca58ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:31,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:31,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:31,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:31,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:15:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,44623,1732504424313 2024-11-25T03:15:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4521b91f 2024-11-25T03:15:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:15:31,318 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:15:31,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:15:31,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:15:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:31,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742159_1335 (size=1097) 2024-11-25T03:15:31,731 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T03:15:31,731 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T03:15:31,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:15:31,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, REOPEN/MOVE}] 2024-11-25T03:15:31,735 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, REOPEN/MOVE 2024-11-25T03:15:31,736 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:31,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, REOPEN/MOVE because future has completed 2024-11-25T03:15:31,738 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:15:31,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:15:31,890 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:31,890 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:15:31,890 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing bcf6a7b706584359872cc54bd64d673e, disabling compactions & flushes 2024-11-25T03:15:31,890 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:31,890 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:31,890 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. after waiting 0 ms 2024-11-25T03:15:31,890 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:31,894 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:15:31,894 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:31,894 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for bcf6a7b706584359872cc54bd64d673e: Waiting for close lock at 1732504531890Running coprocessor pre-close hooks at 1732504531890Disabling compacts and flushes for region at 1732504531890Disabling writes for close at 1732504531890Writing region close event to WAL at 1732504531891 (+1 ms)Running coprocessor post-close hooks at 1732504531894 (+3 ms)Closed at 1732504531894 2024-11-25T03:15:31,894 WARN [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: bcf6a7b706584359872cc54bd64d673e to self. 2024-11-25T03:15:31,896 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:31,897 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=CLOSED 2024-11-25T03:15:31,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:15:31,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-25T03:15:31,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 in 161 msec 2024-11-25T03:15:31,900 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, REOPEN/MOVE; state=CLOSED, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=true 2024-11-25T03:15:32,051 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, REOPEN/MOVE because future has completed 2024-11-25T03:15:32,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:15:32,208 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,208 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:15:32,208 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,208 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:15:32,208 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,208 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,210 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,211 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:32,211 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf6a7b706584359872cc54bd64d673e columnFamilyName A 2024-11-25T03:15:32,212 DEBUG [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:32,212 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(327): Store=bcf6a7b706584359872cc54bd64d673e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:32,213 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,213 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:32,214 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf6a7b706584359872cc54bd64d673e columnFamilyName B 2024-11-25T03:15:32,214 DEBUG [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:32,214 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(327): Store=bcf6a7b706584359872cc54bd64d673e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:32,214 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,215 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:32,215 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf6a7b706584359872cc54bd64d673e columnFamilyName C 2024-11-25T03:15:32,215 DEBUG [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:32,215 INFO [StoreOpener-bcf6a7b706584359872cc54bd64d673e-1 {}] regionserver.HStore(327): Store=bcf6a7b706584359872cc54bd64d673e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:32,215 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,215 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,216 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,216 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,217 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,217 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,218 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:15:32,218 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,219 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened bcf6a7b706584359872cc54bd64d673e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73870861, jitterRate=0.10076160728931427}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:15:32,219 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,220 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for bcf6a7b706584359872cc54bd64d673e: Running coprocessor pre-open hook at 1732504532209Writing region info on filesystem at 1732504532209Initializing all the Stores at 1732504532209Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504532209Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504532209Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504532209Cleaning up temporary data from old regions at 1732504532217 (+8 ms)Running coprocessor post-open hooks at 1732504532219 (+2 ms)Region opened successfully at 1732504532220 (+1 ms) 2024-11-25T03:15:32,220 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., pid=104, masterSystemTime=1732504532205 2024-11-25T03:15:32,222 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,222 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,223 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=OPEN, openSeqNum=5, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:15:32,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-25T03:15:32,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 in 171 msec 2024-11-25T03:15:32,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-25T03:15:32,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, REOPEN/MOVE in 491 msec 2024-11-25T03:15:32,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-25T03:15:32,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-25T03:15:32,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-25T03:15:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-25T03:15:32,231 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:32,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,233 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462aa90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1241f6f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,236 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c5971e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,237 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a655700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,239 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51945852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ae2e044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,242 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5bfc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,243 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746bc630, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,244 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323f0b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393631ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,247 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c05808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4d670d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,250 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685e1e85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293a48e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,252 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfe0a14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42907867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,255 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5e7d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,256 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610f0fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:15:32,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:15:32,257 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:15:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:15:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:15:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c9293b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:15:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:15:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:32,259 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:15:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e13498c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:15:32,259 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,260 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,260 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,260 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:32,260 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,261 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,261 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,261 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,262 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,263 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:15:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:15:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-25T03:15:32,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:15:32,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:15:32,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:15:32,266 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,269 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:15:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:15:32,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:32,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:32,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:32,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:32,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:32,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:32,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125786483e569b44763bfc8c9b215d34b05_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504532282/Put/seqid=0 2024-11-25T03:15:32,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742160_1336 (size=12154) 2024-11-25T03:15:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504592352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504592360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504592360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504532344, max timestamp=1732504532359Num uniques 859; 2024-11-25T03:15:32,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504532344, max timestamp=1732504532359Num uniques 859; 2024-11-25T03:15:32,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:15:32,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:15:32,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732504532344, max timestamp=1732504532359Num uniques 859; 2024-11-25T03:15:32,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:15:32,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:15:32,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:32,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504592429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504592434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504592470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504592471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504592471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504592550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504592550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:15:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:15:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504592690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504592690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504592690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:32,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125786483e569b44763bfc8c9b215d34b05_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125786483e569b44763bfc8c9b215d34b05_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:32,726 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/7c52cd71fde04eb5954d39dca3ba469b, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:32,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/7c52cd71fde04eb5954d39dca3ba469b is 175, key is test_row_0/A:col10/1732504532282/Put/seqid=0 2024-11-25T03:15:32,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:15:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:32,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742161_1337 (size=30955) 2024-11-25T03:15:32,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/7c52cd71fde04eb5954d39dca3ba469b 2024-11-25T03:15:32,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/affb00d9f3e4445b8ff3ef0e8ce193f3 is 50, key is test_row_0/B:col10/1732504532282/Put/seqid=0 2024-11-25T03:15:32,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742162_1338 (size=12001) 2024-11-25T03:15:32,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/affb00d9f3e4445b8ff3ef0e8ce193f3 2024-11-25T03:15:32,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504592770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504592770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:32,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/f4e02283761d43b0a010676a876effac is 50, key is test_row_0/C:col10/1732504532282/Put/seqid=0 2024-11-25T03:15:32,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742163_1339 (size=12001) 2024-11-25T03:15:32,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:15:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:32,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:15:33,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504593010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504593010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504593010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:15:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:33,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504593080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504593090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:15:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:33,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:33,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/f4e02283761d43b0a010676a876effac 2024-11-25T03:15:33,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/7c52cd71fde04eb5954d39dca3ba469b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/7c52cd71fde04eb5954d39dca3ba469b 2024-11-25T03:15:33,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/7c52cd71fde04eb5954d39dca3ba469b, entries=150, sequenceid=14, filesize=30.2 K 2024-11-25T03:15:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/affb00d9f3e4445b8ff3ef0e8ce193f3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/affb00d9f3e4445b8ff3ef0e8ce193f3 2024-11-25T03:15:33,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/affb00d9f3e4445b8ff3ef0e8ce193f3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T03:15:33,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/f4e02283761d43b0a010676a876effac as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f4e02283761d43b0a010676a876effac 2024-11-25T03:15:33,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f4e02283761d43b0a010676a876effac, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T03:15:33,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bcf6a7b706584359872cc54bd64d673e in 928ms, sequenceid=14, compaction requested=false 2024-11-25T03:15:33,211 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T03:15:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:33,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:15:33,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:33,345 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-25T03:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254c2df705cb074c6d8d3c59462d55e9b7_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504532413/Put/seqid=0 2024-11-25T03:15:33,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742164_1340 (size=12154) 2024-11-25T03:15:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:33,360 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254c2df705cb074c6d8d3c59462d55e9b7_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254c2df705cb074c6d8d3c59462d55e9b7_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/e1dc251165c0427f954149189822532e, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/e1dc251165c0427f954149189822532e is 175, key is test_row_0/A:col10/1732504532413/Put/seqid=0 2024-11-25T03:15:33,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742165_1341 (size=30955) 2024-11-25T03:15:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:15:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:33,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:33,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504593549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504593550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504593551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504593600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504593610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504593659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504593670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504593670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,785 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/e1dc251165c0427f954149189822532e 2024-11-25T03:15:33,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/64b10b7f7b5248cb91c4b0109c915d6e is 50, key is test_row_0/B:col10/1732504532413/Put/seqid=0 2024-11-25T03:15:33,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742166_1342 (size=12001) 2024-11-25T03:15:33,884 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:15:33,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504593880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504593889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504593890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,201 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/64b10b7f7b5248cb91c4b0109c915d6e 2024-11-25T03:15:34,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504594190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504594210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504594210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/d64876d5f06242b19a698ae3173b2ccf is 50, key is test_row_0/C:col10/1732504532413/Put/seqid=0 2024-11-25T03:15:34,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742167_1343 (size=12001) 2024-11-25T03:15:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:15:34,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504594620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504594620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,633 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:34,634 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,668 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/d64876d5f06242b19a698ae3173b2ccf 2024-11-25T03:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/e1dc251165c0427f954149189822532e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/e1dc251165c0427f954149189822532e 2024-11-25T03:15:34,676 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/e1dc251165c0427f954149189822532e, entries=150, sequenceid=49, filesize=30.2 K 2024-11-25T03:15:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/64b10b7f7b5248cb91c4b0109c915d6e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/64b10b7f7b5248cb91c4b0109c915d6e 2024-11-25T03:15:34,681 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/64b10b7f7b5248cb91c4b0109c915d6e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:15:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/d64876d5f06242b19a698ae3173b2ccf as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d64876d5f06242b19a698ae3173b2ccf 2024-11-25T03:15:34,688 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d64876d5f06242b19a698ae3173b2ccf, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:15:34,689 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for bcf6a7b706584359872cc54bd64d673e in 1344ms, sequenceid=49, compaction requested=false 2024-11-25T03:15:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-25T03:15:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-25T03:15:34,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-25T03:15:34,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4220 sec 2024-11-25T03:15:34,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.4330 sec 2024-11-25T03:15:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:34,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:15:34,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:34,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:34,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:34,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:34,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:34,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:34,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cb39a7f9f89d4454ba5ffdae371b4892_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504534711/Put/seqid=0 2024-11-25T03:15:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742168_1344 (size=14594) 2024-11-25T03:15:34,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504594785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504534780, max timestamp=1732504534795Num uniques 859; 2024-11-25T03:15:34,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504534780, max timestamp=1732504534795Num uniques 859; 2024-11-25T03:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:34,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504534780, max timestamp=1732504534795Num uniques 859; 2024-11-25T03:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:34,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504594881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504594891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:34,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504594909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504595000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504595000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,123 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:35,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cb39a7f9f89d4454ba5ffdae371b4892_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cb39a7f9f89d4454ba5ffdae371b4892_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:35,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/571f43b39af84a7cbbe7d1950f7bc544, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:35,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/571f43b39af84a7cbbe7d1950f7bc544 is 175, key is test_row_0/A:col10/1732504534711/Put/seqid=0 2024-11-25T03:15:35,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742169_1345 (size=39549) 2024-11-25T03:15:35,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504595130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504595220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504595220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504595450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/571f43b39af84a7cbbe7d1950f7bc544 2024-11-25T03:15:35,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/5f3f9ac779dd4ffb81051d50bfb81b73 is 50, key is test_row_0/B:col10/1732504534711/Put/seqid=0 2024-11-25T03:15:35,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742170_1346 (size=12001) 2024-11-25T03:15:35,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504595540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504595540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:35,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/5f3f9ac779dd4ffb81051d50bfb81b73 2024-11-25T03:15:35,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/2894f88a88d84f118b5e48177e5227eb is 50, key is test_row_0/C:col10/1732504534711/Put/seqid=0 2024-11-25T03:15:35,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742171_1347 (size=12001) 2024-11-25T03:15:35,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504595970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:35,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:36,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504596060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:36,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504596060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:36,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/2894f88a88d84f118b5e48177e5227eb 2024-11-25T03:15:36,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/571f43b39af84a7cbbe7d1950f7bc544 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/571f43b39af84a7cbbe7d1950f7bc544 2024-11-25T03:15:36,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/571f43b39af84a7cbbe7d1950f7bc544, entries=200, sequenceid=60, filesize=38.6 K 2024-11-25T03:15:36,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/5f3f9ac779dd4ffb81051d50bfb81b73 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/5f3f9ac779dd4ffb81051d50bfb81b73 2024-11-25T03:15:36,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/5f3f9ac779dd4ffb81051d50bfb81b73, entries=150, sequenceid=60, filesize=11.7 K 2024-11-25T03:15:36,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/2894f88a88d84f118b5e48177e5227eb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2894f88a88d84f118b5e48177e5227eb 2024-11-25T03:15:36,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2894f88a88d84f118b5e48177e5227eb, entries=150, sequenceid=60, filesize=11.7 K 2024-11-25T03:15:36,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bcf6a7b706584359872cc54bd64d673e in 1665ms, sequenceid=60, compaction requested=true 2024-11-25T03:15:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:36,379 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T03:15:36,380 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:36,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:36,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/A is initiating minor compaction (all files) 2024-11-25T03:15:36,381 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/A in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:36,381 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/7c52cd71fde04eb5954d39dca3ba469b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/e1dc251165c0427f954149189822532e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/571f43b39af84a7cbbe7d1950f7bc544] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=99.1 K 2024-11-25T03:15:36,381 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:36,381 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/7c52cd71fde04eb5954d39dca3ba469b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/e1dc251165c0427f954149189822532e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/571f43b39af84a7cbbe7d1950f7bc544] 2024-11-25T03:15:36,381 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:36,381 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/B is initiating minor compaction (all files) 2024-11-25T03:15:36,381 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/B in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:36,381 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/affb00d9f3e4445b8ff3ef0e8ce193f3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/64b10b7f7b5248cb91c4b0109c915d6e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/5f3f9ac779dd4ffb81051d50bfb81b73] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=35.2 K 2024-11-25T03:15:36,382 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting affb00d9f3e4445b8ff3ef0e8ce193f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732504532276 2024-11-25T03:15:36,382 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c52cd71fde04eb5954d39dca3ba469b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732504532276 2024-11-25T03:15:36,382 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1dc251165c0427f954149189822532e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504532413 2024-11-25T03:15:36,382 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b10b7f7b5248cb91c4b0109c915d6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504532413 2024-11-25T03:15:36,382 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f3f9ac779dd4ffb81051d50bfb81b73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732504533547 2024-11-25T03:15:36,382 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 571f43b39af84a7cbbe7d1950f7bc544, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732504533534 2024-11-25T03:15:36,390 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:36,390 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/e931f7e1799346e4b66966ffe9f44c8d is 50, key is test_row_0/B:col10/1732504534711/Put/seqid=0 2024-11-25T03:15:36,391 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:36,405 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125c3d9a5206ed1426cbd5bffa13b56d798_bcf6a7b706584359872cc54bd64d673e store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:36,407 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125c3d9a5206ed1426cbd5bffa13b56d798_bcf6a7b706584359872cc54bd64d673e, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:36,408 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c3d9a5206ed1426cbd5bffa13b56d798_bcf6a7b706584359872cc54bd64d673e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:36,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742172_1348 (size=12104) 2024-11-25T03:15:36,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742173_1349 (size=4469) 2024-11-25T03:15:36,424 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#A#compaction#298 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:36,425 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c is 175, key is test_row_0/A:col10/1732504534711/Put/seqid=0 2024-11-25T03:15:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:15:36,431 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:36,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:36,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742174_1350 (size=31058) 2024-11-25T03:15:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-25T03:15:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:15:36,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:36,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:36,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:36,441 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c 2024-11-25T03:15:36,447 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/A of bcf6a7b706584359872cc54bd64d673e into 92bc5e33e1c74c33bc1ff9f0ad4daf3c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:36,447 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:36,447 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/A, priority=13, startTime=1732504536379; duration=0sec 2024-11-25T03:15:36,447 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:36,447 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:A 2024-11-25T03:15:36,447 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:36,449 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:36,449 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/C is initiating minor compaction (all files) 2024-11-25T03:15:36,449 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/C in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:36,449 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f4e02283761d43b0a010676a876effac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d64876d5f06242b19a698ae3173b2ccf, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2894f88a88d84f118b5e48177e5227eb] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=35.2 K 2024-11-25T03:15:36,449 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4e02283761d43b0a010676a876effac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732504532276 2024-11-25T03:15:36,450 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64876d5f06242b19a698ae3173b2ccf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732504532413 2024-11-25T03:15:36,451 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2894f88a88d84f118b5e48177e5227eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732504533547 2024-11-25T03:15:36,465 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#C#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:36,465 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a6f4d999736a4a0995b6713b39dc9f6b is 50, key is test_row_0/C:col10/1732504534711/Put/seqid=0 2024-11-25T03:15:36,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742175_1351 (size=12104) 2024-11-25T03:15:36,474 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a6f4d999736a4a0995b6713b39dc9f6b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a6f4d999736a4a0995b6713b39dc9f6b 2024-11-25T03:15:36,479 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/C of bcf6a7b706584359872cc54bd64d673e into a6f4d999736a4a0995b6713b39dc9f6b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:36,479 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:36,479 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/C, priority=13, startTime=1732504536379; duration=0sec 2024-11-25T03:15:36,479 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:36,479 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:C 2024-11-25T03:15:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:15:36,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:15:36,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:36,588 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-25T03:15:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112522d7b0439ec64d5386e114b72aec02ed_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504534879/Put/seqid=0 2024-11-25T03:15:36,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742176_1352 (size=12154) 2024-11-25T03:15:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:36,614 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112522d7b0439ec64d5386e114b72aec02ed_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522d7b0439ec64d5386e114b72aec02ed_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/95824447f9f34bb0b531a6cdf4a9fc4a, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:36,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/95824447f9f34bb0b531a6cdf4a9fc4a is 175, key is test_row_0/A:col10/1732504534879/Put/seqid=0 2024-11-25T03:15:36,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742177_1353 (size=30955) 2024-11-25T03:15:36,620 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/95824447f9f34bb0b531a6cdf4a9fc4a 2024-11-25T03:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/6688b31c303a493aa59c28e5e6702a05 is 50, key is test_row_0/B:col10/1732504534879/Put/seqid=0 2024-11-25T03:15:36,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742178_1354 (size=12001) 2024-11-25T03:15:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:36,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:36,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504596683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:36,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504596689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:15:36,813 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/e931f7e1799346e4b66966ffe9f44c8d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/e931f7e1799346e4b66966ffe9f44c8d 2024-11-25T03:15:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504596810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504596810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:36,819 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/B of bcf6a7b706584359872cc54bd64d673e into e931f7e1799346e4b66966ffe9f44c8d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:36,819 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:36,819 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/B, priority=13, startTime=1732504536379; duration=0sec 2024-11-25T03:15:36,819 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:36,819 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:B 2024-11-25T03:15:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504597000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:37,009 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504597020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504597020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:37,034 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/6688b31c303a493aa59c28e5e6702a05 2024-11-25T03:15:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/7296adfae5b44242bf2766ac10435e5d is 50, key is test_row_0/C:col10/1732504534879/Put/seqid=0 2024-11-25T03:15:37,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742179_1355 (size=12001) 2024-11-25T03:15:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:15:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504597090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504597090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:37,101 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:37,102 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:37,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504597330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:37,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:37,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504597330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:37,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:37,452 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/7296adfae5b44242bf2766ac10435e5d 2024-11-25T03:15:37,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/95824447f9f34bb0b531a6cdf4a9fc4a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/95824447f9f34bb0b531a6cdf4a9fc4a 2024-11-25T03:15:37,459 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/95824447f9f34bb0b531a6cdf4a9fc4a, entries=150, sequenceid=97, filesize=30.2 K 2024-11-25T03:15:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/6688b31c303a493aa59c28e5e6702a05 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/6688b31c303a493aa59c28e5e6702a05 2024-11-25T03:15:37,464 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/6688b31c303a493aa59c28e5e6702a05, entries=150, sequenceid=97, filesize=11.7 K 2024-11-25T03:15:37,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/7296adfae5b44242bf2766ac10435e5d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/7296adfae5b44242bf2766ac10435e5d 2024-11-25T03:15:37,468 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/7296adfae5b44242bf2766ac10435e5d, entries=150, sequenceid=97, filesize=11.7 K 2024-11-25T03:15:37,469 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for bcf6a7b706584359872cc54bd64d673e in 880ms, sequenceid=97, compaction requested=false 2024-11-25T03:15:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-25T03:15:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-25T03:15:37,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-25T03:15:37,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0360 sec 2024-11-25T03:15:37,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0420 sec 2024-11-25T03:15:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:15:37,570 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:37,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-25T03:15:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:15:37,572 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:37,573 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:37,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:15:37,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-25T03:15:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:37,726 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:15:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112502e7968bcc7842f59bf59c0681b87991_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504536681/Put/seqid=0 2024-11-25T03:15:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742180_1356 (size=12154) 2024-11-25T03:15:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:37,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:15:37,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504537893, max timestamp=1732504537937Num uniques 859; 2024-11-25T03:15:37,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504537893, max timestamp=1732504537937Num uniques 859; 2024-11-25T03:15:37,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504537893, max timestamp=1732504537937Num uniques 859; 2024-11-25T03:15:37,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:37,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:37,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:38,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504598001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504598007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504598120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:38,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504598129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,142 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112502e7968bcc7842f59bf59c0681b87991_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112502e7968bcc7842f59bf59c0681b87991_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/39cd76b173324e8e8834b6041884834a, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:38,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/39cd76b173324e8e8834b6041884834a is 175, key is test_row_0/A:col10/1732504536681/Put/seqid=0 2024-11-25T03:15:38,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742181_1357 (size=30955) 2024-11-25T03:15:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:15:38,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504598340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504598350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,548 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/39cd76b173324e8e8834b6041884834a 2024-11-25T03:15:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/f209c5294f474ec7a0e32a039fa30322 is 50, key is test_row_0/B:col10/1732504536681/Put/seqid=0 2024-11-25T03:15:38,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742182_1358 (size=12001) 2024-11-25T03:15:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504598660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504598670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:15:38,959 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/f209c5294f474ec7a0e32a039fa30322 2024-11-25T03:15:38,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/e359d02650d94fae872d23e194bbcb17 is 50, key is test_row_0/C:col10/1732504536681/Put/seqid=0 2024-11-25T03:15:38,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742183_1359 (size=12001) 2024-11-25T03:15:38,969 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/e359d02650d94fae872d23e194bbcb17 2024-11-25T03:15:38,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/39cd76b173324e8e8834b6041884834a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/39cd76b173324e8e8834b6041884834a 2024-11-25T03:15:38,976 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/39cd76b173324e8e8834b6041884834a, entries=150, sequenceid=109, filesize=30.2 K 2024-11-25T03:15:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/f209c5294f474ec7a0e32a039fa30322 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/f209c5294f474ec7a0e32a039fa30322 2024-11-25T03:15:38,981 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/f209c5294f474ec7a0e32a039fa30322, entries=150, sequenceid=109, filesize=11.7 K 2024-11-25T03:15:38,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/e359d02650d94fae872d23e194bbcb17 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e359d02650d94fae872d23e194bbcb17 2024-11-25T03:15:38,985 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e359d02650d94fae872d23e194bbcb17, entries=150, sequenceid=109, filesize=11.7 K 2024-11-25T03:15:38,986 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bcf6a7b706584359872cc54bd64d673e in 1260ms, sequenceid=109, compaction requested=true 2024-11-25T03:15:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-25T03:15:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-25T03:15:38,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-25T03:15:38,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-25T03:15:38,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4220 sec 2024-11-25T03:15:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:39,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-25T03:15:39,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:39,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:39,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:39,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:39,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:39,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:39,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252f4e1d33e7d94193b998b93fe43cdd3f_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504537999/Put/seqid=0 2024-11-25T03:15:39,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742184_1360 (size=14794) 2024-11-25T03:15:39,037 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:39,041 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252f4e1d33e7d94193b998b93fe43cdd3f_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252f4e1d33e7d94193b998b93fe43cdd3f_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:39,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/aa214c6cc60c4ed09a1a4a74f3f38e10, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:39,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/aa214c6cc60c4ed09a1a4a74f3f38e10 is 175, key is test_row_0/A:col10/1732504537999/Put/seqid=0 2024-11-25T03:15:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742185_1361 (size=39749) 2024-11-25T03:15:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504599108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504599120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504599120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,124 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,124 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504599180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504599190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504599230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/aa214c6cc60c4ed09a1a4a74f3f38e10 2024-11-25T03:15:39,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/faf6f6c29f124c95807ef9c9909bc167 is 50, key is test_row_0/B:col10/1732504537999/Put/seqid=0 2024-11-25T03:15:39,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742186_1362 (size=12151) 2024-11-25T03:15:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504599450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:15:39,720 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:39,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-25T03:15:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T03:15:39,722 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:39,723 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:39,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504599770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T03:15:39,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/faf6f6c29f124c95807ef9c9909bc167 2024-11-25T03:15:39,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/417a3219796b44c9bd453ba77f5db7a4 is 50, key is test_row_0/C:col10/1732504537999/Put/seqid=0 2024-11-25T03:15:39,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742187_1363 (size=12151) 2024-11-25T03:15:39,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:39,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:40,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:40,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T03:15:40,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504600190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:40,202 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:40,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504600210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:40,222 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:40,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/417a3219796b44c9bd453ba77f5db7a4 2024-11-25T03:15:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/aa214c6cc60c4ed09a1a4a74f3f38e10 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/aa214c6cc60c4ed09a1a4a74f3f38e10 2024-11-25T03:15:40,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/aa214c6cc60c4ed09a1a4a74f3f38e10, entries=200, sequenceid=145, filesize=38.8 K 2024-11-25T03:15:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/faf6f6c29f124c95807ef9c9909bc167 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/faf6f6c29f124c95807ef9c9909bc167 2024-11-25T03:15:40,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/faf6f6c29f124c95807ef9c9909bc167, entries=150, sequenceid=145, filesize=11.9 K 2024-11-25T03:15:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/417a3219796b44c9bd453ba77f5db7a4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/417a3219796b44c9bd453ba77f5db7a4 2024-11-25T03:15:40,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/417a3219796b44c9bd453ba77f5db7a4, entries=150, sequenceid=145, filesize=11.9 K 2024-11-25T03:15:40,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for bcf6a7b706584359872cc54bd64d673e in 1262ms, sequenceid=145, compaction requested=true 2024-11-25T03:15:40,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:40,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:40,287 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:15:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:40,287 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:15:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:40,288 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:15:40,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:15:40,289 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/A is initiating minor compaction (all files) 2024-11-25T03:15:40,289 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/B is initiating minor compaction (all files) 2024-11-25T03:15:40,289 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/A in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,289 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/B in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,289 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/95824447f9f34bb0b531a6cdf4a9fc4a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/39cd76b173324e8e8834b6041884834a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/aa214c6cc60c4ed09a1a4a74f3f38e10] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=129.6 K 2024-11-25T03:15:40,289 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/e931f7e1799346e4b66966ffe9f44c8d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/6688b31c303a493aa59c28e5e6702a05, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/f209c5294f474ec7a0e32a039fa30322, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/faf6f6c29f124c95807ef9c9909bc167] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=47.1 K 2024-11-25T03:15:40,289 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,289 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/95824447f9f34bb0b531a6cdf4a9fc4a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/39cd76b173324e8e8834b6041884834a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/aa214c6cc60c4ed09a1a4a74f3f38e10] 2024-11-25T03:15:40,289 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e931f7e1799346e4b66966ffe9f44c8d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732504533547 2024-11-25T03:15:40,289 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92bc5e33e1c74c33bc1ff9f0ad4daf3c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732504533547 2024-11-25T03:15:40,289 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 6688b31c303a493aa59c28e5e6702a05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732504534863 2024-11-25T03:15:40,289 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95824447f9f34bb0b531a6cdf4a9fc4a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732504534863 2024-11-25T03:15:40,290 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f209c5294f474ec7a0e32a039fa30322, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504536669 2024-11-25T03:15:40,290 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39cd76b173324e8e8834b6041884834a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504536669 2024-11-25T03:15:40,290 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting faf6f6c29f124c95807ef9c9909bc167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732504537999 2024-11-25T03:15:40,291 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa214c6cc60c4ed09a1a4a74f3f38e10, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732504537998 2024-11-25T03:15:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:40,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:15:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:40,305 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:40,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255aab18a5f6d24f72b6689bb56aefbd52_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504540291/Put/seqid=0 2024-11-25T03:15:40,307 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#B#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:40,307 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/a76b83fce8a740a4b9f530857680bbf4 is 50, key is test_row_0/B:col10/1732504537999/Put/seqid=0 2024-11-25T03:15:40,318 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125910ec799e57745779de10954839f59a1_bcf6a7b706584359872cc54bd64d673e store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:40,320 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125910ec799e57745779de10954839f59a1_bcf6a7b706584359872cc54bd64d673e, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:40,320 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125910ec799e57745779de10954839f59a1_bcf6a7b706584359872cc54bd64d673e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:40,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742188_1364 (size=14794) 2024-11-25T03:15:40,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742189_1365 (size=12391) 2024-11-25T03:15:40,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742190_1366 (size=4469) 2024-11-25T03:15:40,328 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#A#compaction#309 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:40,328 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/869def0a6e0948588334d3c193bcbe2c is 175, key is test_row_0/A:col10/1732504537999/Put/seqid=0 2024-11-25T03:15:40,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742191_1367 (size=31345) 2024-11-25T03:15:40,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T03:15:40,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:40,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:40,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504540412, max timestamp=1732504540515Num uniques 859; 2024-11-25T03:15:40,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:40,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504540412, max timestamp=1732504540515Num uniques 859; 2024-11-25T03:15:40,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:40,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504540412, max timestamp=1732504540515Num uniques 859; 2024-11-25T03:15:40,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:40,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504600696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:40,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:40,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255aab18a5f6d24f72b6689bb56aefbd52_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aab18a5f6d24f72b6689bb56aefbd52_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:40,726 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e9ad845400b49aeb7b6f408785ddfca, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:40,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e9ad845400b49aeb7b6f408785ddfca is 175, key is test_row_0/A:col10/1732504540291/Put/seqid=0 2024-11-25T03:15:40,731 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/a76b83fce8a740a4b9f530857680bbf4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/a76b83fce8a740a4b9f530857680bbf4 2024-11-25T03:15:40,737 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/B of bcf6a7b706584359872cc54bd64d673e into a76b83fce8a740a4b9f530857680bbf4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:40,737 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:40,737 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/B, priority=12, startTime=1732504540287; duration=0sec 2024-11-25T03:15:40,737 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:40,737 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:B 2024-11-25T03:15:40,737 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:15:40,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742192_1368 (size=39749) 2024-11-25T03:15:40,739 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:15:40,739 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/C is initiating minor compaction (all files) 2024-11-25T03:15:40,739 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/C in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,739 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a6f4d999736a4a0995b6713b39dc9f6b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/7296adfae5b44242bf2766ac10435e5d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e359d02650d94fae872d23e194bbcb17, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/417a3219796b44c9bd453ba77f5db7a4] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=47.1 K 2024-11-25T03:15:40,740 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/869def0a6e0948588334d3c193bcbe2c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/869def0a6e0948588334d3c193bcbe2c 2024-11-25T03:15:40,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e9ad845400b49aeb7b6f408785ddfca 2024-11-25T03:15:40,740 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a6f4d999736a4a0995b6713b39dc9f6b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732504533547 2024-11-25T03:15:40,741 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 7296adfae5b44242bf2766ac10435e5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732504534863 2024-11-25T03:15:40,741 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e359d02650d94fae872d23e194bbcb17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732504536669 2024-11-25T03:15:40,742 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 417a3219796b44c9bd453ba77f5db7a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732504537999 2024-11-25T03:15:40,745 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/A of bcf6a7b706584359872cc54bd64d673e into 869def0a6e0948588334d3c193bcbe2c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:40,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:40,745 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/A, priority=12, startTime=1732504540286; duration=0sec 2024-11-25T03:15:40,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:40,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:A 2024-11-25T03:15:40,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/eae9eaf5f9154d9dabd5a32e9cf246d7 is 50, key is test_row_0/B:col10/1732504540291/Put/seqid=0 2024-11-25T03:15:40,751 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#C#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:40,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742193_1369 (size=12151) 2024-11-25T03:15:40,752 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/2eb7ca3d42794f3888959e09f9cfc0a0 is 50, key is test_row_0/C:col10/1732504537999/Put/seqid=0 2024-11-25T03:15:40,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742194_1370 (size=12391) 2024-11-25T03:15:40,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:40,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:40,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504600820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T03:15:40,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:40,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504601040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:41,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:41,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:41,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/eae9eaf5f9154d9dabd5a32e9cf246d7 2024-11-25T03:15:41,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/baa4fd8a569a47da9135eb85627de60a is 50, key is test_row_0/C:col10/1732504540291/Put/seqid=0 2024-11-25T03:15:41,161 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/2eb7ca3d42794f3888959e09f9cfc0a0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2eb7ca3d42794f3888959e09f9cfc0a0 2024-11-25T03:15:41,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742195_1371 (size=12151) 2024-11-25T03:15:41,168 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/C of bcf6a7b706584359872cc54bd64d673e into 2eb7ca3d42794f3888959e09f9cfc0a0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:41,168 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:41,169 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/C, priority=12, startTime=1732504540287; duration=0sec 2024-11-25T03:15:41,169 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:41,169 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:C 2024-11-25T03:15:41,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504601360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:41,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:41,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/baa4fd8a569a47da9135eb85627de60a 2024-11-25T03:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e9ad845400b49aeb7b6f408785ddfca as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e9ad845400b49aeb7b6f408785ddfca 2024-11-25T03:15:41,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e9ad845400b49aeb7b6f408785ddfca, entries=200, sequenceid=156, filesize=38.8 K 2024-11-25T03:15:41,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/eae9eaf5f9154d9dabd5a32e9cf246d7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/eae9eaf5f9154d9dabd5a32e9cf246d7 2024-11-25T03:15:41,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/eae9eaf5f9154d9dabd5a32e9cf246d7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-25T03:15:41,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/baa4fd8a569a47da9135eb85627de60a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/baa4fd8a569a47da9135eb85627de60a 2024-11-25T03:15:41,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/baa4fd8a569a47da9135eb85627de60a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-25T03:15:41,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bcf6a7b706584359872cc54bd64d673e in 1290ms, sequenceid=156, compaction requested=false 2024-11-25T03:15:41,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:41,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T03:15:41,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:41,713 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-25T03:15:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:41,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de802ff83d9d44ee933e2073ac350237_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504540685/Put/seqid=0 2024-11-25T03:15:41,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742196_1372 (size=12304) 2024-11-25T03:15:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T03:15:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:41,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:41,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504601955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504602070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:42,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:42,129 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de802ff83d9d44ee933e2073ac350237_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de802ff83d9d44ee933e2073ac350237_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:42,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4b8b6ad375124c99ada6b94d99b7a42f, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:42,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4b8b6ad375124c99ada6b94d99b7a42f is 175, key is test_row_0/A:col10/1732504540685/Put/seqid=0 2024-11-25T03:15:42,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742197_1373 (size=31105) 2024-11-25T03:15:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504602210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:42,220 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:42,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504602249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:42,262 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:42,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504602290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:42,541 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4b8b6ad375124c99ada6b94d99b7a42f 2024-11-25T03:15:42,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/23aebf78c0c5478e963006fd604c6ce4 is 50, key is test_row_0/B:col10/1732504540685/Put/seqid=0 2024-11-25T03:15:42,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742198_1374 (size=12151) 2024-11-25T03:15:42,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504602600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:42,953 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/23aebf78c0c5478e963006fd604c6ce4 2024-11-25T03:15:42,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/03f02082b7f64d81b2769a38f2993dce is 50, key is test_row_0/C:col10/1732504540685/Put/seqid=0 2024-11-25T03:15:42,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742199_1375 (size=12151) 2024-11-25T03:15:43,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504603110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:43,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1732504603130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:43,140 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1732504603170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:43,175 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:43,187 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:15:43,365 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/03f02082b7f64d81b2769a38f2993dce 2024-11-25T03:15:43,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4b8b6ad375124c99ada6b94d99b7a42f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4b8b6ad375124c99ada6b94d99b7a42f 2024-11-25T03:15:43,374 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4b8b6ad375124c99ada6b94d99b7a42f, entries=150, sequenceid=195, filesize=30.4 K 2024-11-25T03:15:43,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/23aebf78c0c5478e963006fd604c6ce4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/23aebf78c0c5478e963006fd604c6ce4 2024-11-25T03:15:43,378 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/23aebf78c0c5478e963006fd604c6ce4, entries=150, sequenceid=195, filesize=11.9 K 2024-11-25T03:15:43,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/03f02082b7f64d81b2769a38f2993dce as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/03f02082b7f64d81b2769a38f2993dce 2024-11-25T03:15:43,382 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/03f02082b7f64d81b2769a38f2993dce, entries=150, sequenceid=195, filesize=11.9 K 2024-11-25T03:15:43,383 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for bcf6a7b706584359872cc54bd64d673e in 1670ms, sequenceid=195, compaction requested=true 2024-11-25T03:15:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-25T03:15:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-25T03:15:43,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-25T03:15:43,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6620 sec 2024-11-25T03:15:43,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.6660 sec 2024-11-25T03:15:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T03:15:43,890 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:43,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-25T03:15:43,892 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:15:43,893 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:43,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:15:44,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T03:15:44,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:44,046 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:15:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:44,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258ed9a5748cb54c64a3c8df3df7d104b2_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504541938/Put/seqid=0 2024-11-25T03:15:44,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742200_1376 (size=12304) 2024-11-25T03:15:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:44,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:15:44,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504544249, max timestamp=1732504544278Num uniques 859; 2024-11-25T03:15:44,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504544249, max timestamp=1732504544278Num uniques 859; 2024-11-25T03:15:44,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:44,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:44,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504544249, max timestamp=1732504544278Num uniques 859; 2024-11-25T03:15:44,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:44,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504604440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:44,464 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258ed9a5748cb54c64a3c8df3df7d104b2_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258ed9a5748cb54c64a3c8df3df7d104b2_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/09c428195b854a4c92cf56a91896df1c, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/09c428195b854a4c92cf56a91896df1c is 175, key is test_row_0/A:col10/1732504541938/Put/seqid=0 2024-11-25T03:15:44,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742201_1377 (size=31105) 2024-11-25T03:15:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:15:44,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504604560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:44,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504604780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:44,870 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/09c428195b854a4c92cf56a91896df1c 2024-11-25T03:15:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/fc87074b4be447f88753260247535c91 is 50, key is test_row_0/B:col10/1732504541938/Put/seqid=0 2024-11-25T03:15:44,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742202_1378 (size=12151) 2024-11-25T03:15:44,881 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/fc87074b4be447f88753260247535c91 2024-11-25T03:15:44,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/e35b2ed78320480cb884bd28e98b0afa is 50, key is test_row_0/C:col10/1732504541938/Put/seqid=0 2024-11-25T03:15:44,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742203_1379 (size=12151) 2024-11-25T03:15:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:15:45,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504605100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:45,291 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/e35b2ed78320480cb884bd28e98b0afa 2024-11-25T03:15:45,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/09c428195b854a4c92cf56a91896df1c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/09c428195b854a4c92cf56a91896df1c 2024-11-25T03:15:45,300 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/09c428195b854a4c92cf56a91896df1c, entries=150, sequenceid=206, filesize=30.4 K 2024-11-25T03:15:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/fc87074b4be447f88753260247535c91 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/fc87074b4be447f88753260247535c91 2024-11-25T03:15:45,304 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/fc87074b4be447f88753260247535c91, entries=150, sequenceid=206, filesize=11.9 K 2024-11-25T03:15:45,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/e35b2ed78320480cb884bd28e98b0afa as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e35b2ed78320480cb884bd28e98b0afa 2024-11-25T03:15:45,308 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e35b2ed78320480cb884bd28e98b0afa, entries=150, sequenceid=206, filesize=11.9 K 2024-11-25T03:15:45,309 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bcf6a7b706584359872cc54bd64d673e in 1263ms, sequenceid=206, compaction requested=true 2024-11-25T03:15:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-25T03:15:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-25T03:15:45,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-25T03:15:45,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-25T03:15:45,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.4220 sec 2024-11-25T03:15:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:45,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-25T03:15:45,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:45,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:45,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:45,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:45,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:45,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:45,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125741a89b2508c4e8ebeee20fdac4ff054_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504545625/Put/seqid=0 2024-11-25T03:15:45,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742204_1380 (size=14794) 2024-11-25T03:15:45,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504605689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:45,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504605800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:46,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504606020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:46,039 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:15:46,040 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:46,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-25T03:15:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T03:15:46,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125741a89b2508c4e8ebeee20fdac4ff054_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125741a89b2508c4e8ebeee20fdac4ff054_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:46,043 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:46,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:46,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/cbe510c783b8472485c414e44929b794, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:46,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:46,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/cbe510c783b8472485c414e44929b794 is 175, key is test_row_0/A:col10/1732504545625/Put/seqid=0 2024-11-25T03:15:46,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742205_1381 (size=39749) 2024-11-25T03:15:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T03:15:46,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33182 deadline: 1732504606260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:46,272 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:46,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1732504606280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:46,290 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in bcf6a7b706584359872cc54bd64d673e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:15:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:46,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504606330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:46,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:46,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:46,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T03:15:46,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/cbe510c783b8472485c414e44929b794 2024-11-25T03:15:46,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/67bf23e41a2d47ca9335e1d079493bdc is 50, key is test_row_0/B:col10/1732504545625/Put/seqid=0 2024-11-25T03:15:46,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742206_1382 (size=12151) 2024-11-25T03:15:46,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T03:15:46,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:46,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:46,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504606840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:46,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/67bf23e41a2d47ca9335e1d079493bdc 2024-11-25T03:15:46,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/d02a3296c7ac4a8c8c6dc584dc48c5e1 is 50, key is test_row_0/C:col10/1732504545625/Put/seqid=0 2024-11-25T03:15:46,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742207_1383 (size=12151) 2024-11-25T03:15:46,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:46,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:46,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:46,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:47,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T03:15:47,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:47,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:47,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:47,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:47,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/d02a3296c7ac4a8c8c6dc584dc48c5e1 2024-11-25T03:15:47,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/cbe510c783b8472485c414e44929b794 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/cbe510c783b8472485c414e44929b794 2024-11-25T03:15:47,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/cbe510c783b8472485c414e44929b794, entries=200, sequenceid=243, filesize=38.8 K 2024-11-25T03:15:47,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/67bf23e41a2d47ca9335e1d079493bdc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/67bf23e41a2d47ca9335e1d079493bdc 2024-11-25T03:15:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/67bf23e41a2d47ca9335e1d079493bdc, entries=150, sequenceid=243, filesize=11.9 K 2024-11-25T03:15:47,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/d02a3296c7ac4a8c8c6dc584dc48c5e1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d02a3296c7ac4a8c8c6dc584dc48c5e1 2024-11-25T03:15:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d02a3296c7ac4a8c8c6dc584dc48c5e1, entries=150, sequenceid=243, filesize=11.9 K 2024-11-25T03:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for bcf6a7b706584359872cc54bd64d673e in 1672ms, sequenceid=243, compaction requested=true 2024-11-25T03:15:47,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,299 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T03:15:47,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:47,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:47,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:47,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:47,299 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T03:15:47,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,301 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173053 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T03:15:47,301 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/A is initiating minor compaction (all files) 2024-11-25T03:15:47,301 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/A in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,301 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/869def0a6e0948588334d3c193bcbe2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e9ad845400b49aeb7b6f408785ddfca, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4b8b6ad375124c99ada6b94d99b7a42f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/09c428195b854a4c92cf56a91896df1c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/cbe510c783b8472485c414e44929b794] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=169.0 K 2024-11-25T03:15:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,301 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,301 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/869def0a6e0948588334d3c193bcbe2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e9ad845400b49aeb7b6f408785ddfca, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4b8b6ad375124c99ada6b94d99b7a42f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/09c428195b854a4c92cf56a91896df1c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/cbe510c783b8472485c414e44929b794] 2024-11-25T03:15:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,302 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60995 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T03:15:47,302 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/B is initiating minor compaction (all files) 2024-11-25T03:15:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,302 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/B in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,302 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/a76b83fce8a740a4b9f530857680bbf4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/eae9eaf5f9154d9dabd5a32e9cf246d7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/23aebf78c0c5478e963006fd604c6ce4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/fc87074b4be447f88753260247535c91, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/67bf23e41a2d47ca9335e1d079493bdc] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=59.6 K 2024-11-25T03:15:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,302 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 869def0a6e0948588334d3c193bcbe2c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732504537999 2024-11-25T03:15:47,302 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a76b83fce8a740a4b9f530857680bbf4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732504537999 2024-11-25T03:15:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,303 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting eae9eaf5f9154d9dabd5a32e9cf246d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732504539061 2024-11-25T03:15:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,303 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9ad845400b49aeb7b6f408785ddfca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732504539061 2024-11-25T03:15:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,303 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 23aebf78c0c5478e963006fd604c6ce4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732504540676 2024-11-25T03:15:47,303 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8b6ad375124c99ada6b94d99b7a42f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732504540676 2024-11-25T03:15:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,304 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c428195b854a4c92cf56a91896df1c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732504541938 2024-11-25T03:15:47,304 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting fc87074b4be447f88753260247535c91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732504541938 2024-11-25T03:15:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,304 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bf23e41a2d47ca9335e1d079493bdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732504544378 2024-11-25T03:15:47,304 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbe510c783b8472485c414e44929b794, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732504544378 2024-11-25T03:15:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,319 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:15:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,319 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/1ed344c9b10f424f9388a193ea8e977f is 50, key is test_row_0/B:col10/1732504545625/Put/seqid=0 2024-11-25T03:15:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,320 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,322 DEBUG [regionserver/38a5d2077268:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-11-25T03:15:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,325 WARN [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-25T03:15:47,325 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125a1e2dbab165f4b86879cd27220309cda_bcf6a7b706584359872cc54bd64d673e store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,327 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/7c52cd71fde04eb5954d39dca3ba469b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/e1dc251165c0427f954149189822532e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/571f43b39af84a7cbbe7d1950f7bc544, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/95824447f9f34bb0b531a6cdf4a9fc4a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/39cd76b173324e8e8834b6041884834a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/aa214c6cc60c4ed09a1a4a74f3f38e10] to archive 2024-11-25T03:15:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,328 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125a1e2dbab165f4b86879cd27220309cda_bcf6a7b706584359872cc54bd64d673e, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,328 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a1e2dbab165f4b86879cd27220309cda_bcf6a7b706584359872cc54bd64d673e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,331 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/7c52cd71fde04eb5954d39dca3ba469b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/7c52cd71fde04eb5954d39dca3ba469b 2024-11-25T03:15:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/e1dc251165c0427f954149189822532e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/e1dc251165c0427f954149189822532e 2024-11-25T03:15:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742208_1384 (size=12561) 2024-11-25T03:15:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,340 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/571f43b39af84a7cbbe7d1950f7bc544 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/571f43b39af84a7cbbe7d1950f7bc544 2024-11-25T03:15:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/92bc5e33e1c74c33bc1ff9f0ad4daf3c 2024-11-25T03:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,343 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/95824447f9f34bb0b531a6cdf4a9fc4a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/95824447f9f34bb0b531a6cdf4a9fc4a 2024-11-25T03:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/39cd76b173324e8e8834b6041884834a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/39cd76b173324e8e8834b6041884834a 2024-11-25T03:15:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,345 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/aa214c6cc60c4ed09a1a4a74f3f38e10 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/aa214c6cc60c4ed09a1a4a74f3f38e10 2024-11-25T03:15:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,347 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/affb00d9f3e4445b8ff3ef0e8ce193f3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/64b10b7f7b5248cb91c4b0109c915d6e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/e931f7e1799346e4b66966ffe9f44c8d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/5f3f9ac779dd4ffb81051d50bfb81b73, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/6688b31c303a493aa59c28e5e6702a05, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/f209c5294f474ec7a0e32a039fa30322, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/faf6f6c29f124c95807ef9c9909bc167] to archive 2024-11-25T03:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742209_1385 (size=4469) 2024-11-25T03:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,351 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/affb00d9f3e4445b8ff3ef0e8ce193f3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/affb00d9f3e4445b8ff3ef0e8ce193f3 2024-11-25T03:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,352 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#A#compaction#325 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,353 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/f507813148934097b2c1416e178ff9ce is 175, key is test_row_0/A:col10/1732504545625/Put/seqid=0 2024-11-25T03:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/64b10b7f7b5248cb91c4b0109c915d6e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/64b10b7f7b5248cb91c4b0109c915d6e 2024-11-25T03:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,357 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/e931f7e1799346e4b66966ffe9f44c8d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/e931f7e1799346e4b66966ffe9f44c8d 2024-11-25T03:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,358 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/5f3f9ac779dd4ffb81051d50bfb81b73 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/5f3f9ac779dd4ffb81051d50bfb81b73 2024-11-25T03:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,359 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/6688b31c303a493aa59c28e5e6702a05 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/6688b31c303a493aa59c28e5e6702a05 2024-11-25T03:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,360 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/f209c5294f474ec7a0e32a039fa30322 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/f209c5294f474ec7a0e32a039fa30322 2024-11-25T03:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,362 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/faf6f6c29f124c95807ef9c9909bc167 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/faf6f6c29f124c95807ef9c9909bc167 2024-11-25T03:15:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742210_1386 (size=31277) 2024-11-25T03:15:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,364 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f4e02283761d43b0a010676a876effac, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d64876d5f06242b19a698ae3173b2ccf, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a6f4d999736a4a0995b6713b39dc9f6b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2894f88a88d84f118b5e48177e5227eb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/7296adfae5b44242bf2766ac10435e5d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e359d02650d94fae872d23e194bbcb17, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/417a3219796b44c9bd453ba77f5db7a4] to archive 2024-11-25T03:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,365 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,367 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f4e02283761d43b0a010676a876effac to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f4e02283761d43b0a010676a876effac 2024-11-25T03:15:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,368 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/f507813148934097b2c1416e178ff9ce as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f507813148934097b2c1416e178ff9ce 2024-11-25T03:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,369 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d64876d5f06242b19a698ae3173b2ccf to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d64876d5f06242b19a698ae3173b2ccf 2024-11-25T03:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,372 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a6f4d999736a4a0995b6713b39dc9f6b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a6f4d999736a4a0995b6713b39dc9f6b 2024-11-25T03:15:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,374 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2894f88a88d84f118b5e48177e5227eb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2894f88a88d84f118b5e48177e5227eb 2024-11-25T03:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,376 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/7296adfae5b44242bf2766ac10435e5d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/7296adfae5b44242bf2766ac10435e5d 2024-11-25T03:15:47,376 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/A of bcf6a7b706584359872cc54bd64d673e into f507813148934097b2c1416e178ff9ce(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:47,376 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:47,376 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/A, priority=11, startTime=1732504547299; duration=0sec 2024-11-25T03:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,376 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:47,376 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:A 2024-11-25T03:15:47,377 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T03:15:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,377 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e359d02650d94fae872d23e194bbcb17 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e359d02650d94fae872d23e194bbcb17 2024-11-25T03:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,378 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60995 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T03:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,378 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/C is initiating minor compaction (all files) 2024-11-25T03:15:47,378 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/C in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,379 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2eb7ca3d42794f3888959e09f9cfc0a0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/baa4fd8a569a47da9135eb85627de60a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/03f02082b7f64d81b2769a38f2993dce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e35b2ed78320480cb884bd28e98b0afa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d02a3296c7ac4a8c8c6dc584dc48c5e1] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=59.6 K 2024-11-25T03:15:47,379 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/417a3219796b44c9bd453ba77f5db7a4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/417a3219796b44c9bd453ba77f5db7a4 2024-11-25T03:15:47,379 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb7ca3d42794f3888959e09f9cfc0a0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732504537999 2024-11-25T03:15:47,379 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa4fd8a569a47da9135eb85627de60a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732504539061 2024-11-25T03:15:47,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03f02082b7f64d81b2769a38f2993dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732504540676 2024-11-25T03:15:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting e35b2ed78320480cb884bd28e98b0afa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732504541938 2024-11-25T03:15:47,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02a3296c7ac4a8c8c6dc584dc48c5e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732504544378 2024-11-25T03:15:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,390 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,391 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/5429014d63eb47bfa7fb7e75cd2163f7 is 50, key is test_row_0/C:col10/1732504545625/Put/seqid=0 2024-11-25T03:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742211_1387 (size=12323) 2024-11-25T03:15:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T03:15:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,422 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:15:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ea50d4e1d7f84e86ab04f7df3673260a_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_1/A:col10/1732504545646/Put/seqid=0 2024-11-25T03:15:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742212_1388 (size=9814) 2024-11-25T03:15:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,440 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ea50d4e1d7f84e86ab04f7df3673260a_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ea50d4e1d7f84e86ab04f7df3673260a_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e4148de052a4cfebb5a948bc3b47b37, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e4148de052a4cfebb5a948bc3b47b37 is 175, key is test_row_1/A:col10/1732504545646/Put/seqid=0 2024-11-25T03:15:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742213_1389 (size=22461) 2024-11-25T03:15:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,744 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/1ed344c9b10f424f9388a193ea8e977f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/1ed344c9b10f424f9388a193ea8e977f 2024-11-25T03:15:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,749 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/B of bcf6a7b706584359872cc54bd64d673e into 1ed344c9b10f424f9388a193ea8e977f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,749 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:47,749 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/B, priority=11, startTime=1732504547299; duration=0sec 2024-11-25T03:15:47,749 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:47,749 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:B 2024-11-25T03:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,801 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/5429014d63eb47bfa7fb7e75cd2163f7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/5429014d63eb47bfa7fb7e75cd2163f7 2024-11-25T03:15:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,807 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/C of bcf6a7b706584359872cc54bd64d673e into 5429014d63eb47bfa7fb7e75cd2163f7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,807 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:47,808 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/C, priority=11, startTime=1732504547299; duration=0sec 2024-11-25T03:15:47,808 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:47,808 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:C 2024-11-25T03:15:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,863 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e4148de052a4cfebb5a948bc3b47b37 2024-11-25T03:15:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/9f7c02d903d347058a076db08ba3aa63 is 50, key is test_row_1/B:col10/1732504545646/Put/seqid=0 2024-11-25T03:15:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742214_1390 (size=9757) 2024-11-25T03:15:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,876 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/9f7c02d903d347058a076db08ba3aa63 2024-11-25T03:15:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/376ecb47a6f044f9876fe861178af039 is 50, key is test_row_1/C:col10/1732504545646/Put/seqid=0 2024-11-25T03:15:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742215_1391 (size=9757) 2024-11-25T03:15:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,898 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/376ecb47a6f044f9876fe861178af039 2024-11-25T03:15:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4e4148de052a4cfebb5a948bc3b47b37 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e4148de052a4cfebb5a948bc3b47b37 2024-11-25T03:15:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,910 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e4148de052a4cfebb5a948bc3b47b37, entries=100, sequenceid=254, filesize=21.9 K 2024-11-25T03:15:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/9f7c02d903d347058a076db08ba3aa63 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/9f7c02d903d347058a076db08ba3aa63 2024-11-25T03:15:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,917 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/9f7c02d903d347058a076db08ba3aa63, entries=100, sequenceid=254, filesize=9.5 K 2024-11-25T03:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/376ecb47a6f044f9876fe861178af039 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/376ecb47a6f044f9876fe861178af039 2024-11-25T03:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,923 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/376ecb47a6f044f9876fe861178af039, entries=100, sequenceid=254, filesize=9.5 K 2024-11-25T03:15:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,924 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for bcf6a7b706584359872cc54bd64d673e in 502ms, sequenceid=254, compaction requested=false 2024-11-25T03:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-25T03:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-25T03:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-25T03:15:47,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-25T03:15:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.8890 sec 2024-11-25T03:15:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:48,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:15:48,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:48,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:48,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:48,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:48,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:48,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:48,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252244f41acb644c6fb1634369161c83da_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504547974/Put/seqid=0 2024-11-25T03:15:48,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742216_1392 (size=12454) 2024-11-25T03:15:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T03:15:48,190 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:48,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:15:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-25T03:15:48,193 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:15:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:15:48,194 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:15:48,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:15:48,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504548204, max timestamp=1732504548246Num uniques 859; 2024-11-25T03:15:48,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504548204, max timestamp=1732504548246Num uniques 859; 2024-11-25T03:15:48,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:48,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:48,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504548204, max timestamp=1732504548246Num uniques 859; 2024-11-25T03:15:48,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:15:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:15:48,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:48,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:48,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504608433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:48,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:48,499 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252244f41acb644c6fb1634369161c83da_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252244f41acb644c6fb1634369161c83da_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:48,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:48,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:48,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/8572644aec2c46b2848db7977c8cb883, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:48,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/8572644aec2c46b2848db7977c8cb883 is 175, key is test_row_0/A:col10/1732504547974/Put/seqid=0 2024-11-25T03:15:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:15:48,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742217_1393 (size=31255) 2024-11-25T03:15:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504608550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:48,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504608770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:48,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:15:48,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/8572644aec2c46b2848db7977c8cb883 2024-11-25T03:15:48,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/be9f2f2140c44646966901c592cc4fd5 is 50, key is test_row_0/B:col10/1732504547974/Put/seqid=0 2024-11-25T03:15:48,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742218_1394 (size=12301) 2024-11-25T03:15:48,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:48,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:48,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:48,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504609080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:49,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:49,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:49,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/be9f2f2140c44646966901c592cc4fd5 2024-11-25T03:15:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:15:49,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/48c0764f15684f4f9a63ba2c014c772a is 50, key is test_row_0/C:col10/1732504547974/Put/seqid=0 2024-11-25T03:15:49,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742219_1395 (size=12301) 2024-11-25T03:15:49,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504609600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:49,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/48c0764f15684f4f9a63ba2c014c772a 2024-11-25T03:15:49,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:15:49,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/8572644aec2c46b2848db7977c8cb883 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8572644aec2c46b2848db7977c8cb883 2024-11-25T03:15:49,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8572644aec2c46b2848db7977c8cb883, entries=150, sequenceid=266, filesize=30.5 K 2024-11-25T03:15:49,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/be9f2f2140c44646966901c592cc4fd5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/be9f2f2140c44646966901c592cc4fd5 2024-11-25T03:15:49,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/be9f2f2140c44646966901c592cc4fd5, entries=150, sequenceid=266, filesize=12.0 K 2024-11-25T03:15:49,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/48c0764f15684f4f9a63ba2c014c772a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/48c0764f15684f4f9a63ba2c014c772a 2024-11-25T03:15:49,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/48c0764f15684f4f9a63ba2c014c772a, entries=150, sequenceid=266, filesize=12.0 K 2024-11-25T03:15:49,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bcf6a7b706584359872cc54bd64d673e in 1721ms, sequenceid=266, compaction requested=true 2024-11-25T03:15:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:49,752 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:49,752 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:49,753 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:49,753 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:49,753 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/A is initiating minor compaction (all files) 2024-11-25T03:15:49,753 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/B is initiating minor compaction (all files) 2024-11-25T03:15:49,753 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/A in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,753 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/B in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,753 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f507813148934097b2c1416e178ff9ce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e4148de052a4cfebb5a948bc3b47b37, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8572644aec2c46b2848db7977c8cb883] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=83.0 K 2024-11-25T03:15:49,753 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,753 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/1ed344c9b10f424f9388a193ea8e977f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/9f7c02d903d347058a076db08ba3aa63, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/be9f2f2140c44646966901c592cc4fd5] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=33.8 K 2024-11-25T03:15:49,753 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f507813148934097b2c1416e178ff9ce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e4148de052a4cfebb5a948bc3b47b37, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8572644aec2c46b2848db7977c8cb883] 2024-11-25T03:15:49,754 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f507813148934097b2c1416e178ff9ce, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732504544378 2024-11-25T03:15:49,754 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed344c9b10f424f9388a193ea8e977f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732504544378 2024-11-25T03:15:49,754 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7c02d903d347058a076db08ba3aa63, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732504545646 2024-11-25T03:15:49,754 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4148de052a4cfebb5a948bc3b47b37, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732504545646 2024-11-25T03:15:49,754 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting be9f2f2140c44646966901c592cc4fd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732504547974 2024-11-25T03:15:49,754 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8572644aec2c46b2848db7977c8cb883, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732504547974 2024-11-25T03:15:49,759 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:49,760 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:49,761 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/486ab7b1737b4b45b2d4c1863812935d is 50, key is test_row_0/B:col10/1732504547974/Put/seqid=0 2024-11-25T03:15:49,761 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125b6dad808f19f48c6aab5e77e421a43a8_bcf6a7b706584359872cc54bd64d673e store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:49,762 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125b6dad808f19f48c6aab5e77e421a43a8_bcf6a7b706584359872cc54bd64d673e, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:49,763 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b6dad808f19f48c6aab5e77e421a43a8_bcf6a7b706584359872cc54bd64d673e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:49,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742220_1396 (size=12575) 2024-11-25T03:15:49,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742221_1397 (size=4469) 2024-11-25T03:15:49,772 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/486ab7b1737b4b45b2d4c1863812935d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/486ab7b1737b4b45b2d4c1863812935d 2024-11-25T03:15:49,779 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/B of bcf6a7b706584359872cc54bd64d673e into 486ab7b1737b4b45b2d4c1863812935d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:49,779 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:49,779 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/B, priority=13, startTime=1732504549752; duration=0sec 2024-11-25T03:15:49,780 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:49,780 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:B 2024-11-25T03:15:49,780 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:49,780 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:49,780 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/C is initiating minor compaction (all files) 2024-11-25T03:15:49,781 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/C in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,781 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/5429014d63eb47bfa7fb7e75cd2163f7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/376ecb47a6f044f9876fe861178af039, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/48c0764f15684f4f9a63ba2c014c772a] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=33.6 K 2024-11-25T03:15:49,781 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5429014d63eb47bfa7fb7e75cd2163f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732504544378 2024-11-25T03:15:49,781 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 376ecb47a6f044f9876fe861178af039, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732504545646 2024-11-25T03:15:49,781 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 48c0764f15684f4f9a63ba2c014c772a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732504547974 2024-11-25T03:15:49,788 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:49,788 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/486ca47cae2f44a4ab36112eda8d0397 is 50, key is test_row_0/C:col10/1732504547974/Put/seqid=0 2024-11-25T03:15:49,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742222_1398 (size=12575) 2024-11-25T03:15:49,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:15:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:49,891 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-25T03:15:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258afe931ac787455daa5617cd5521f661_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504548377/Put/seqid=0 2024-11-25T03:15:49,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742223_1399 (size=12454) 2024-11-25T03:15:49,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:49,908 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258afe931ac787455daa5617cd5521f661_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258afe931ac787455daa5617cd5521f661_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/f025219a463b4882b9260fcff5bd4a96, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:49,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/f025219a463b4882b9260fcff5bd4a96 is 175, key is test_row_0/A:col10/1732504548377/Put/seqid=0 2024-11-25T03:15:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742224_1400 (size=31255) 2024-11-25T03:15:49,919 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/f025219a463b4882b9260fcff5bd4a96 2024-11-25T03:15:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/17fe52bfc70c42cbb71cd586d3aacd22 is 50, key is test_row_0/B:col10/1732504548377/Put/seqid=0 2024-11-25T03:15:49,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742225_1401 (size=12301) 2024-11-25T03:15:50,169 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:50,170 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/689f906adf644a17a3d93b975730e359 is 175, key is test_row_0/A:col10/1732504547974/Put/seqid=0 2024-11-25T03:15:50,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742226_1402 (size=31529) 2024-11-25T03:15:50,195 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/486ca47cae2f44a4ab36112eda8d0397 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/486ca47cae2f44a4ab36112eda8d0397 2024-11-25T03:15:50,199 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/C of bcf6a7b706584359872cc54bd64d673e into 486ca47cae2f44a4ab36112eda8d0397(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:50,199 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:50,199 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/C, priority=13, startTime=1732504549752; duration=0sec 2024-11-25T03:15:50,199 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:50,199 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:C 2024-11-25T03:15:50,340 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/17fe52bfc70c42cbb71cd586d3aacd22 2024-11-25T03:15:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:15:50,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a9039789553443cfa254398366a6e6f4 is 50, key is test_row_0/C:col10/1732504548377/Put/seqid=0 2024-11-25T03:15:50,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742227_1403 (size=12301) 2024-11-25T03:15:50,579 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/689f906adf644a17a3d93b975730e359 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/689f906adf644a17a3d93b975730e359 2024-11-25T03:15:50,584 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/A of bcf6a7b706584359872cc54bd64d673e into 689f906adf644a17a3d93b975730e359(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:50,584 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:50,584 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/A, priority=13, startTime=1732504549752; duration=0sec 2024-11-25T03:15:50,584 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:50,584 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:A 2024-11-25T03:15:50,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. as already flushing 2024-11-25T03:15:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:50,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504610679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:50,760 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a9039789553443cfa254398366a6e6f4 2024-11-25T03:15:50,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/f025219a463b4882b9260fcff5bd4a96 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f025219a463b4882b9260fcff5bd4a96 2024-11-25T03:15:50,768 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f025219a463b4882b9260fcff5bd4a96, entries=150, sequenceid=304, filesize=30.5 K 2024-11-25T03:15:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/17fe52bfc70c42cbb71cd586d3aacd22 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/17fe52bfc70c42cbb71cd586d3aacd22 2024-11-25T03:15:50,773 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/17fe52bfc70c42cbb71cd586d3aacd22, entries=150, sequenceid=304, filesize=12.0 K 2024-11-25T03:15:50,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a9039789553443cfa254398366a6e6f4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a9039789553443cfa254398366a6e6f4 2024-11-25T03:15:50,777 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a9039789553443cfa254398366a6e6f4, entries=150, sequenceid=304, filesize=12.0 K 2024-11-25T03:15:50,780 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for bcf6a7b706584359872cc54bd64d673e in 889ms, sequenceid=304, compaction requested=false 2024-11-25T03:15:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-25T03:15:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-25T03:15:50,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-25T03:15:50,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5870 sec 2024-11-25T03:15:50,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5920 sec 2024-11-25T03:15:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:50,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:15:50,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:50,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:50,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:50,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:50,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:50,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:50,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254459cfaab9d44e74a159de186cf19298_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504550658/Put/seqid=0 2024-11-25T03:15:50,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742228_1404 (size=14994) 2024-11-25T03:15:50,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:50,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254459cfaab9d44e74a159de186cf19298_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254459cfaab9d44e74a159de186cf19298_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:50,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/d5f88cfe1b994d4ea980e631d652e00b, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:50,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/d5f88cfe1b994d4ea980e631d652e00b is 175, key is test_row_0/A:col10/1732504550658/Put/seqid=0 2024-11-25T03:15:50,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742229_1405 (size=39949) 2024-11-25T03:15:50,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504550871, max timestamp=1732504550942Num uniques 859; 2024-11-25T03:15:50,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504550871, max timestamp=1732504550942Num uniques 859; 2024-11-25T03:15:50,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504550871, max timestamp=1732504550942Num uniques 859; 2024-11-25T03:15:50,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:50,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:50,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:15:51,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504611083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504611200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:51,240 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/d5f88cfe1b994d4ea980e631d652e00b 2024-11-25T03:15:51,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/951cd4f54f2e47a3b2636bfd58d5e625 is 50, key is test_row_0/B:col10/1732504550658/Put/seqid=0 2024-11-25T03:15:51,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742230_1406 (size=12301) 2024-11-25T03:15:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504611420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:51,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/951cd4f54f2e47a3b2636bfd58d5e625 2024-11-25T03:15:51,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/f1f8fe9ad02a46bba4736b0126c2f58c is 50, key is test_row_0/C:col10/1732504550658/Put/seqid=0 2024-11-25T03:15:51,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742231_1407 (size=12301) 2024-11-25T03:15:51,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:15:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1732504611740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 2024-11-25T03:15:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:15:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:15:52,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/f1f8fe9ad02a46bba4736b0126c2f58c 2024-11-25T03:15:52,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/d5f88cfe1b994d4ea980e631d652e00b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/d5f88cfe1b994d4ea980e631d652e00b 2024-11-25T03:15:52,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/d5f88cfe1b994d4ea980e631d652e00b, entries=200, sequenceid=317, filesize=39.0 K 2024-11-25T03:15:52,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/951cd4f54f2e47a3b2636bfd58d5e625 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/951cd4f54f2e47a3b2636bfd58d5e625 2024-11-25T03:15:52,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/951cd4f54f2e47a3b2636bfd58d5e625, entries=150, sequenceid=317, filesize=12.0 K 2024-11-25T03:15:52,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/f1f8fe9ad02a46bba4736b0126c2f58c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f1f8fe9ad02a46bba4736b0126c2f58c 2024-11-25T03:15:52,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f1f8fe9ad02a46bba4736b0126c2f58c, entries=150, sequenceid=317, filesize=12.0 K 2024-11-25T03:15:52,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bcf6a7b706584359872cc54bd64d673e in 1289ms, sequenceid=317, compaction requested=true 2024-11-25T03:15:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:15:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:15:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bcf6a7b706584359872cc54bd64d673e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:15:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:52,082 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:52,082 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:52,083 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:52,083 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:52,083 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/B is initiating minor compaction (all files) 2024-11-25T03:15:52,083 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/A is initiating minor compaction (all files) 2024-11-25T03:15:52,083 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/B in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:52,083 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/A in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:52,083 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/486ab7b1737b4b45b2d4c1863812935d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/17fe52bfc70c42cbb71cd586d3aacd22, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/951cd4f54f2e47a3b2636bfd58d5e625] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=36.3 K 2024-11-25T03:15:52,083 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/689f906adf644a17a3d93b975730e359, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f025219a463b4882b9260fcff5bd4a96, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/d5f88cfe1b994d4ea980e631d652e00b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=100.3 K 2024-11-25T03:15:52,083 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:52,083 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/689f906adf644a17a3d93b975730e359, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f025219a463b4882b9260fcff5bd4a96, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/d5f88cfe1b994d4ea980e631d652e00b] 2024-11-25T03:15:52,083 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 486ab7b1737b4b45b2d4c1863812935d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732504547974 2024-11-25T03:15:52,084 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 689f906adf644a17a3d93b975730e359, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732504547974 2024-11-25T03:15:52,084 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 17fe52bfc70c42cbb71cd586d3aacd22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732504548377 2024-11-25T03:15:52,084 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f025219a463b4882b9260fcff5bd4a96, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732504548377 2024-11-25T03:15:52,084 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 951cd4f54f2e47a3b2636bfd58d5e625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732504550658 2024-11-25T03:15:52,084 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f88cfe1b994d4ea980e631d652e00b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732504550658 2024-11-25T03:15:52,090 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:52,091 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:52,091 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/2d9b2d2207fb4472a8122e0b7eb05241 is 50, key is test_row_0/B:col10/1732504550658/Put/seqid=0 2024-11-25T03:15:52,092 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411253892818b7f234063803f6bf248e03d69_bcf6a7b706584359872cc54bd64d673e store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:52,094 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411253892818b7f234063803f6bf248e03d69_bcf6a7b706584359872cc54bd64d673e, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:52,094 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253892818b7f234063803f6bf248e03d69_bcf6a7b706584359872cc54bd64d673e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:52,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742233_1409 (size=4469) 2024-11-25T03:15:52,098 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#A#compaction#343 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:52,098 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/8e7ea1f2caa3433fa8d5bb75ad0ce49b is 175, key is test_row_0/A:col10/1732504550658/Put/seqid=0 2024-11-25T03:15:52,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742232_1408 (size=12677) 2024-11-25T03:15:52,106 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/2d9b2d2207fb4472a8122e0b7eb05241 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/2d9b2d2207fb4472a8122e0b7eb05241 2024-11-25T03:15:52,110 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/B of bcf6a7b706584359872cc54bd64d673e into 2d9b2d2207fb4472a8122e0b7eb05241(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:52,110 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:52,110 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/B, priority=13, startTime=1732504552082; duration=0sec 2024-11-25T03:15:52,110 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:15:52,110 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:B 2024-11-25T03:15:52,110 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:15:52,111 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:15:52,111 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): bcf6a7b706584359872cc54bd64d673e/C is initiating minor compaction (all files) 2024-11-25T03:15:52,111 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bcf6a7b706584359872cc54bd64d673e/C in TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:52,111 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/486ca47cae2f44a4ab36112eda8d0397, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a9039789553443cfa254398366a6e6f4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f1f8fe9ad02a46bba4736b0126c2f58c] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp, totalSize=36.3 K 2024-11-25T03:15:52,112 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 486ca47cae2f44a4ab36112eda8d0397, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732504547974 2024-11-25T03:15:52,112 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a9039789553443cfa254398366a6e6f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732504548377 2024-11-25T03:15:52,112 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f8fe9ad02a46bba4736b0126c2f58c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732504550658 2024-11-25T03:15:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742234_1410 (size=31631) 2024-11-25T03:15:52,123 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bcf6a7b706584359872cc54bd64d673e#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:15:52,124 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/4e0500cc616040eb882fed116765b761 is 50, key is test_row_0/C:col10/1732504550658/Put/seqid=0 2024-11-25T03:15:52,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742235_1411 (size=12677) 2024-11-25T03:15:52,264 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-25T03:15:52,264 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:52,264 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,264 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:52,266 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-25T03:15:52,266 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:52,266 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,266 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:52,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-25T03:15:52,272 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-25T03:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:52,272 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:52,272 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:15:52,272 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:52,273 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-25T03:15:52,273 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:52,273 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,273 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:52,276 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-25T03:15:52,276 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:52,276 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,276 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,276 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-25T03:15:52,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:52,276 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:52,276 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,276 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:52,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:52,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ac83123a59294b5196b9941170a06af3_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504551048/Put/seqid=0 2024-11-25T03:15:52,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742236_1412 (size=14994) 2024-11-25T03:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:15:52,370 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:52,526 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/8e7ea1f2caa3433fa8d5bb75ad0ce49b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8e7ea1f2caa3433fa8d5bb75ad0ce49b 2024-11-25T03:15:52,530 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/A of bcf6a7b706584359872cc54bd64d673e into 8e7ea1f2caa3433fa8d5bb75ad0ce49b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:52,530 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:52,530 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/A, priority=13, startTime=1732504552082; duration=0sec 2024-11-25T03:15:52,530 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:52,530 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:A 2024-11-25T03:15:52,535 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/4e0500cc616040eb882fed116765b761 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/4e0500cc616040eb882fed116765b761 2024-11-25T03:15:52,539 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bcf6a7b706584359872cc54bd64d673e/C of bcf6a7b706584359872cc54bd64d673e into 4e0500cc616040eb882fed116765b761(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:15:52,539 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:52,539 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., storeName=bcf6a7b706584359872cc54bd64d673e/C, priority=13, startTime=1732504552082; duration=0sec 2024-11-25T03:15:52,539 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:15:52,539 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bcf6a7b706584359872cc54bd64d673e:C 2024-11-25T03:15:52,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:52,690 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ac83123a59294b5196b9941170a06af3_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ac83123a59294b5196b9941170a06af3_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:52,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4a4fe7dda4dd40aaa2fa8684f272ad68, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:52,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4a4fe7dda4dd40aaa2fa8684f272ad68 is 175, key is test_row_0/A:col10/1732504551048/Put/seqid=0 2024-11-25T03:15:52,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742237_1413 (size=39949) 2024-11-25T03:15:53,095 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4a4fe7dda4dd40aaa2fa8684f272ad68 2024-11-25T03:15:53,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/2516fd72fd3641f8a6c2c28375919963 is 50, key is test_row_0/B:col10/1732504551048/Put/seqid=0 2024-11-25T03:15:53,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742238_1414 (size=12301) 2024-11-25T03:15:53,162 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-25T03:15:53,162 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:53,162 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:53,162 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:53,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:53,261 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-25T03:15:53,262 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:53,262 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:53,262 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:53,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:53,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/2516fd72fd3641f8a6c2c28375919963 2024-11-25T03:15:53,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/abb81cd1065f4cb0874f9b9639d7df6a is 50, key is test_row_0/C:col10/1732504551048/Put/seqid=0 2024-11-25T03:15:53,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742239_1415 (size=12301) 2024-11-25T03:15:53,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/abb81cd1065f4cb0874f9b9639d7df6a 2024-11-25T03:15:53,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/4a4fe7dda4dd40aaa2fa8684f272ad68 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4a4fe7dda4dd40aaa2fa8684f272ad68 2024-11-25T03:15:53,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4a4fe7dda4dd40aaa2fa8684f272ad68, entries=200, sequenceid=354, filesize=39.0 K 2024-11-25T03:15:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/2516fd72fd3641f8a6c2c28375919963 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/2516fd72fd3641f8a6c2c28375919963 2024-11-25T03:15:53,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/2516fd72fd3641f8a6c2c28375919963, entries=150, sequenceid=354, filesize=12.0 K 2024-11-25T03:15:53,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/abb81cd1065f4cb0874f9b9639d7df6a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/abb81cd1065f4cb0874f9b9639d7df6a 2024-11-25T03:15:53,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/abb81cd1065f4cb0874f9b9639d7df6a, entries=150, sequenceid=354, filesize=12.0 K 2024-11-25T03:15:53,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=13.42 KB/13740 for bcf6a7b706584359872cc54bd64d673e in 1657ms, sequenceid=354, compaction requested=false 2024-11-25T03:15:53,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bcf6a7b706584359872cc54bd64d673e: 2024-11-25T03:15:56,302 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-11-25T03:15:56,302 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:56,302 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:56,302 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:56,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:56,391 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-25T03:15:56,392 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:15:56,392 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:56,392 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:56,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 185 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 916 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2745 rows 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 919 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2753 rows 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 922 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2764 rows 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 925 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2768 rows 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 922 2024-11-25T03:15:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2763 rows 2024-11-25T03:15:56,392 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:15:56,392 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:15:56,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:56,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:15:56,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:15:56,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:15:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:15:56,396 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504556396"}]},"ts":"1732504556396"} 2024-11-25T03:15:56,397 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:15:56,397 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:15:56,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:15:56,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, UNASSIGN}] 2024-11-25T03:15:56,399 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, UNASSIGN 2024-11-25T03:15:56,400 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:56,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, UNASSIGN because future has completed 2024-11-25T03:15:56,401 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:15:56,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:15:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:15:56,553 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:56,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:15:56,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing bcf6a7b706584359872cc54bd64d673e, disabling compactions & flushes 2024-11-25T03:15:56,553 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:56,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:56,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. after waiting 0 ms 2024-11-25T03:15:56,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:56,554 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing bcf6a7b706584359872cc54bd64d673e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-25T03:15:56,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=A 2024-11-25T03:15:56,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:56,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=B 2024-11-25T03:15:56,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:56,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bcf6a7b706584359872cc54bd64d673e, store=C 2024-11-25T03:15:56,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:15:56,559 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254a757a4b85ea447f8b3156378ce839fb_bcf6a7b706584359872cc54bd64d673e is 50, key is test_row_0/A:col10/1732504556390/Put/seqid=0 2024-11-25T03:15:56,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742240_1416 (size=12454) 2024-11-25T03:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:15:56,962 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:56,966 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254a757a4b85ea447f8b3156378ce839fb_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254a757a4b85ea447f8b3156378ce839fb_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:56,967 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/ef86000ded9349dbbacc192720787147, store: [table=TestAcidGuarantees family=A region=bcf6a7b706584359872cc54bd64d673e] 2024-11-25T03:15:56,967 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/ef86000ded9349dbbacc192720787147 is 175, key is test_row_0/A:col10/1732504556390/Put/seqid=0 2024-11-25T03:15:56,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742241_1417 (size=31255) 2024-11-25T03:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:15:57,371 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/ef86000ded9349dbbacc192720787147 2024-11-25T03:15:57,376 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/847a1787f495441db13ad100ad0d119a is 50, key is test_row_0/B:col10/1732504556390/Put/seqid=0 2024-11-25T03:15:57,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742242_1418 (size=12301) 2024-11-25T03:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:15:57,780 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/847a1787f495441db13ad100ad0d119a 2024-11-25T03:15:57,786 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a55bb86baa594406a5b47aa25f73f0ab is 50, key is test_row_0/C:col10/1732504556390/Put/seqid=0 2024-11-25T03:15:57,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742243_1419 (size=12301) 2024-11-25T03:15:58,189 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a55bb86baa594406a5b47aa25f73f0ab 2024-11-25T03:15:58,193 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/A/ef86000ded9349dbbacc192720787147 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/ef86000ded9349dbbacc192720787147 2024-11-25T03:15:58,196 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/ef86000ded9349dbbacc192720787147, entries=150, sequenceid=363, filesize=30.5 K 2024-11-25T03:15:58,197 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/B/847a1787f495441db13ad100ad0d119a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/847a1787f495441db13ad100ad0d119a 2024-11-25T03:15:58,199 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/847a1787f495441db13ad100ad0d119a, entries=150, sequenceid=363, filesize=12.0 K 2024-11-25T03:15:58,200 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/.tmp/C/a55bb86baa594406a5b47aa25f73f0ab as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a55bb86baa594406a5b47aa25f73f0ab 2024-11-25T03:15:58,202 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a55bb86baa594406a5b47aa25f73f0ab, entries=150, sequenceid=363, filesize=12.0 K 2024-11-25T03:15:58,203 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for bcf6a7b706584359872cc54bd64d673e in 1650ms, sequenceid=363, compaction requested=true 2024-11-25T03:15:58,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/869def0a6e0948588334d3c193bcbe2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e9ad845400b49aeb7b6f408785ddfca, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4b8b6ad375124c99ada6b94d99b7a42f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/09c428195b854a4c92cf56a91896df1c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/cbe510c783b8472485c414e44929b794, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f507813148934097b2c1416e178ff9ce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e4148de052a4cfebb5a948bc3b47b37, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/689f906adf644a17a3d93b975730e359, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8572644aec2c46b2848db7977c8cb883, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f025219a463b4882b9260fcff5bd4a96, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/d5f88cfe1b994d4ea980e631d652e00b] to archive 2024-11-25T03:15:58,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:58,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/869def0a6e0948588334d3c193bcbe2c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/869def0a6e0948588334d3c193bcbe2c 2024-11-25T03:15:58,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e9ad845400b49aeb7b6f408785ddfca to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e9ad845400b49aeb7b6f408785ddfca 2024-11-25T03:15:58,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4b8b6ad375124c99ada6b94d99b7a42f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4b8b6ad375124c99ada6b94d99b7a42f 2024-11-25T03:15:58,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/09c428195b854a4c92cf56a91896df1c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/09c428195b854a4c92cf56a91896df1c 2024-11-25T03:15:58,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/cbe510c783b8472485c414e44929b794 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/cbe510c783b8472485c414e44929b794 2024-11-25T03:15:58,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f507813148934097b2c1416e178ff9ce to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f507813148934097b2c1416e178ff9ce 2024-11-25T03:15:58,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e4148de052a4cfebb5a948bc3b47b37 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4e4148de052a4cfebb5a948bc3b47b37 2024-11-25T03:15:58,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/689f906adf644a17a3d93b975730e359 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/689f906adf644a17a3d93b975730e359 2024-11-25T03:15:58,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8572644aec2c46b2848db7977c8cb883 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8572644aec2c46b2848db7977c8cb883 2024-11-25T03:15:58,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f025219a463b4882b9260fcff5bd4a96 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/f025219a463b4882b9260fcff5bd4a96 2024-11-25T03:15:58,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/d5f88cfe1b994d4ea980e631d652e00b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/d5f88cfe1b994d4ea980e631d652e00b 2024-11-25T03:15:58,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/a76b83fce8a740a4b9f530857680bbf4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/eae9eaf5f9154d9dabd5a32e9cf246d7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/23aebf78c0c5478e963006fd604c6ce4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/fc87074b4be447f88753260247535c91, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/1ed344c9b10f424f9388a193ea8e977f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/67bf23e41a2d47ca9335e1d079493bdc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/9f7c02d903d347058a076db08ba3aa63, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/486ab7b1737b4b45b2d4c1863812935d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/be9f2f2140c44646966901c592cc4fd5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/17fe52bfc70c42cbb71cd586d3aacd22, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/951cd4f54f2e47a3b2636bfd58d5e625] to archive 2024-11-25T03:15:58,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:58,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/a76b83fce8a740a4b9f530857680bbf4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/a76b83fce8a740a4b9f530857680bbf4 2024-11-25T03:15:58,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/eae9eaf5f9154d9dabd5a32e9cf246d7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/eae9eaf5f9154d9dabd5a32e9cf246d7 2024-11-25T03:15:58,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/23aebf78c0c5478e963006fd604c6ce4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/23aebf78c0c5478e963006fd604c6ce4 2024-11-25T03:15:58,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/fc87074b4be447f88753260247535c91 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/fc87074b4be447f88753260247535c91 2024-11-25T03:15:58,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/1ed344c9b10f424f9388a193ea8e977f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/1ed344c9b10f424f9388a193ea8e977f 2024-11-25T03:15:58,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/67bf23e41a2d47ca9335e1d079493bdc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/67bf23e41a2d47ca9335e1d079493bdc 2024-11-25T03:15:58,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/9f7c02d903d347058a076db08ba3aa63 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/9f7c02d903d347058a076db08ba3aa63 2024-11-25T03:15:58,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/486ab7b1737b4b45b2d4c1863812935d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/486ab7b1737b4b45b2d4c1863812935d 2024-11-25T03:15:58,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/be9f2f2140c44646966901c592cc4fd5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/be9f2f2140c44646966901c592cc4fd5 2024-11-25T03:15:58,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/17fe52bfc70c42cbb71cd586d3aacd22 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/17fe52bfc70c42cbb71cd586d3aacd22 2024-11-25T03:15:58,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/951cd4f54f2e47a3b2636bfd58d5e625 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/951cd4f54f2e47a3b2636bfd58d5e625 2024-11-25T03:15:58,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2eb7ca3d42794f3888959e09f9cfc0a0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/baa4fd8a569a47da9135eb85627de60a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/03f02082b7f64d81b2769a38f2993dce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e35b2ed78320480cb884bd28e98b0afa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/5429014d63eb47bfa7fb7e75cd2163f7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d02a3296c7ac4a8c8c6dc584dc48c5e1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/376ecb47a6f044f9876fe861178af039, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/486ca47cae2f44a4ab36112eda8d0397, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/48c0764f15684f4f9a63ba2c014c772a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a9039789553443cfa254398366a6e6f4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f1f8fe9ad02a46bba4736b0126c2f58c] to archive 2024-11-25T03:15:58,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:15:58,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2eb7ca3d42794f3888959e09f9cfc0a0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/2eb7ca3d42794f3888959e09f9cfc0a0 2024-11-25T03:15:58,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/baa4fd8a569a47da9135eb85627de60a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/baa4fd8a569a47da9135eb85627de60a 2024-11-25T03:15:58,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/03f02082b7f64d81b2769a38f2993dce to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/03f02082b7f64d81b2769a38f2993dce 2024-11-25T03:15:58,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e35b2ed78320480cb884bd28e98b0afa to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/e35b2ed78320480cb884bd28e98b0afa 2024-11-25T03:15:58,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/5429014d63eb47bfa7fb7e75cd2163f7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/5429014d63eb47bfa7fb7e75cd2163f7 2024-11-25T03:15:58,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d02a3296c7ac4a8c8c6dc584dc48c5e1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/d02a3296c7ac4a8c8c6dc584dc48c5e1 2024-11-25T03:15:58,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/376ecb47a6f044f9876fe861178af039 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/376ecb47a6f044f9876fe861178af039 2024-11-25T03:15:58,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/486ca47cae2f44a4ab36112eda8d0397 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/486ca47cae2f44a4ab36112eda8d0397 2024-11-25T03:15:58,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/48c0764f15684f4f9a63ba2c014c772a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/48c0764f15684f4f9a63ba2c014c772a 2024-11-25T03:15:58,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a9039789553443cfa254398366a6e6f4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a9039789553443cfa254398366a6e6f4 2024-11-25T03:15:58,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f1f8fe9ad02a46bba4736b0126c2f58c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/f1f8fe9ad02a46bba4736b0126c2f58c 2024-11-25T03:15:58,238 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=4 2024-11-25T03:15:58,238 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. 2024-11-25T03:15:58,238 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for bcf6a7b706584359872cc54bd64d673e: Waiting for close lock at 1732504556553Running coprocessor pre-close hooks at 1732504556553Disabling compacts and flushes for region at 1732504556553Disabling writes for close at 1732504556553Obtaining lock to block concurrent updates at 1732504556554 (+1 ms)Preparing flush snapshotting stores in bcf6a7b706584359872cc54bd64d673e at 1732504556554Finished memstore snapshotting TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732504556554Flushing stores of TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e. at 1732504556554Flushing bcf6a7b706584359872cc54bd64d673e/A: creating writer at 1732504556554Flushing mob file bcf6a7b706584359872cc54bd64d673e/A: appending metadata at 1732504556559 (+5 ms)Flushing mob file bcf6a7b706584359872cc54bd64d673e/A: closing flushed file at 1732504556559Flushing bcf6a7b706584359872cc54bd64d673e/A: appending metadata at 1732504556967 (+408 ms)Flushing bcf6a7b706584359872cc54bd64d673e/A: closing flushed file at 1732504556967Flushing bcf6a7b706584359872cc54bd64d673e/B: creating writer at 1732504557374 (+407 ms)Flushing bcf6a7b706584359872cc54bd64d673e/B: appending metadata at 1732504557376 (+2 ms)Flushing bcf6a7b706584359872cc54bd64d673e/B: closing flushed file at 1732504557376Flushing bcf6a7b706584359872cc54bd64d673e/C: creating writer at 1732504557783 (+407 ms)Flushing bcf6a7b706584359872cc54bd64d673e/C: appending metadata at 1732504557785 (+2 ms)Flushing bcf6a7b706584359872cc54bd64d673e/C: closing flushed file at 1732504557785Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7466e34: reopening flushed file at 1732504558192 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e95b8bd: reopening flushed file at 1732504558196 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39427c5: reopening flushed file at 1732504558199 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for bcf6a7b706584359872cc54bd64d673e in 1650ms, sequenceid=363, compaction requested=true at 1732504558203 (+4 ms)Writing region close event to WAL at 1732504558235 (+32 ms)Running coprocessor post-close hooks at 1732504558238 (+3 ms)Closed at 1732504558238 2024-11-25T03:15:58,240 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,241 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=bcf6a7b706584359872cc54bd64d673e, regionState=CLOSED 2024-11-25T03:15:58,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:15:58,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-25T03:15:58,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure bcf6a7b706584359872cc54bd64d673e, server=38a5d2077268,43357,1732504425108 in 1.8410 sec 2024-11-25T03:15:58,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-25T03:15:58,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bcf6a7b706584359872cc54bd64d673e, UNASSIGN in 1.8450 sec 2024-11-25T03:15:58,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-25T03:15:58,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8480 sec 2024-11-25T03:15:58,248 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504558247"}]},"ts":"1732504558247"} 2024-11-25T03:15:58,249 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:15:58,249 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:15:58,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8560 sec 2024-11-25T03:15:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:15:58,540 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:58,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:58,543 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T03:15:58,543 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:58,546 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,547 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/recovered.edits] 2024-11-25T03:15:58,549 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4a4fe7dda4dd40aaa2fa8684f272ad68 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/4a4fe7dda4dd40aaa2fa8684f272ad68 2024-11-25T03:15:58,550 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8e7ea1f2caa3433fa8d5bb75ad0ce49b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/8e7ea1f2caa3433fa8d5bb75ad0ce49b 2024-11-25T03:15:58,551 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/ef86000ded9349dbbacc192720787147 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/A/ef86000ded9349dbbacc192720787147 2024-11-25T03:15:58,553 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/2516fd72fd3641f8a6c2c28375919963 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/2516fd72fd3641f8a6c2c28375919963 2024-11-25T03:15:58,554 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/2d9b2d2207fb4472a8122e0b7eb05241 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/2d9b2d2207fb4472a8122e0b7eb05241 2024-11-25T03:15:58,554 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/847a1787f495441db13ad100ad0d119a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/B/847a1787f495441db13ad100ad0d119a 2024-11-25T03:15:58,556 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/4e0500cc616040eb882fed116765b761 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/4e0500cc616040eb882fed116765b761 2024-11-25T03:15:58,556 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a55bb86baa594406a5b47aa25f73f0ab to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/a55bb86baa594406a5b47aa25f73f0ab 2024-11-25T03:15:58,557 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/abb81cd1065f4cb0874f9b9639d7df6a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/C/abb81cd1065f4cb0874f9b9639d7df6a 2024-11-25T03:15:58,559 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/recovered.edits/366.seqid to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e/recovered.edits/366.seqid 2024-11-25T03:15:58,559 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,559 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:15:58,560 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:15:58,560 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T03:15:58,562 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112502e7968bcc7842f59bf59c0681b87991_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112502e7968bcc7842f59bf59c0681b87991_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,563 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252244f41acb644c6fb1634369161c83da_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252244f41acb644c6fb1634369161c83da_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,564 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522d7b0439ec64d5386e114b72aec02ed_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522d7b0439ec64d5386e114b72aec02ed_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,565 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252f4e1d33e7d94193b998b93fe43cdd3f_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252f4e1d33e7d94193b998b93fe43cdd3f_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,566 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254459cfaab9d44e74a159de186cf19298_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254459cfaab9d44e74a159de186cf19298_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,567 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254a757a4b85ea447f8b3156378ce839fb_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254a757a4b85ea447f8b3156378ce839fb_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,568 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254c2df705cb074c6d8d3c59462d55e9b7_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254c2df705cb074c6d8d3c59462d55e9b7_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,569 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aab18a5f6d24f72b6689bb56aefbd52_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aab18a5f6d24f72b6689bb56aefbd52_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,570 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125741a89b2508c4e8ebeee20fdac4ff054_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125741a89b2508c4e8ebeee20fdac4ff054_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,571 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125786483e569b44763bfc8c9b215d34b05_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125786483e569b44763bfc8c9b215d34b05_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,572 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258afe931ac787455daa5617cd5521f661_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258afe931ac787455daa5617cd5521f661_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,573 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258ed9a5748cb54c64a3c8df3df7d104b2_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258ed9a5748cb54c64a3c8df3df7d104b2_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,574 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ac83123a59294b5196b9941170a06af3_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ac83123a59294b5196b9941170a06af3_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,575 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cb39a7f9f89d4454ba5ffdae371b4892_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cb39a7f9f89d4454ba5ffdae371b4892_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,575 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de802ff83d9d44ee933e2073ac350237_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de802ff83d9d44ee933e2073ac350237_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,576 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ea50d4e1d7f84e86ab04f7df3673260a_bcf6a7b706584359872cc54bd64d673e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ea50d4e1d7f84e86ab04f7df3673260a_bcf6a7b706584359872cc54bd64d673e 2024-11-25T03:15:58,577 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:15:58,578 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:58,580 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:15:58,582 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:15:58,583 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:58,583 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:15:58,583 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732504558583"}]},"ts":"9223372036854775807"} 2024-11-25T03:15:58,585 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:15:58,585 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bcf6a7b706584359872cc54bd64d673e, NAME => 'TestAcidGuarantees,,1732504529159.bcf6a7b706584359872cc54bd64d673e.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:15:58,585 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:15:58,585 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732504558585"}]},"ts":"9223372036854775807"} 2024-11-25T03:15:58,586 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:15:58,587 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:58,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-25T03:15:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T03:15:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:15:58,650 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:15:58,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 247) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-34767401_22 at /127.0.0.1:57678 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 417), ProcessCount=11 (was 11), AvailableMemoryMB=6375 (was 6469) 2024-11-25T03:15:58,668 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=6375 2024-11-25T03:15:58,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:15:58,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:15:58,670 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:15:58,671 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:58,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-25T03:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-25T03:15:58,671 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:15:58,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742244_1420 (size=1061) 2024-11-25T03:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-25T03:15:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-25T03:15:59,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:15:59,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742245_1421 (size=53) 2024-11-25T03:15:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-25T03:15:59,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:15:59,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d74b21d2d7105b5e1e755662f34cb2a2, disabling compactions & flushes 2024-11-25T03:15:59,483 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. after waiting 0 ms 2024-11-25T03:15:59,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,483 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d74b21d2d7105b5e1e755662f34cb2a2: Waiting for close lock at 1732504559483Disabling compacts and flushes for region at 1732504559483Disabling writes for close at 1732504559483Writing region close event to WAL at 1732504559483Closed at 1732504559483 2024-11-25T03:15:59,484 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:15:59,484 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732504559484"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732504559484"}]},"ts":"1732504559484"} 2024-11-25T03:15:59,485 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:15:59,486 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:15:59,486 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504559486"}]},"ts":"1732504559486"} 2024-11-25T03:15:59,487 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:15:59,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, ASSIGN}] 2024-11-25T03:15:59,488 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, ASSIGN 2024-11-25T03:15:59,489 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, ASSIGN; state=OFFLINE, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=false 2024-11-25T03:15:59,639 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=d74b21d2d7105b5e1e755662f34cb2a2, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:59,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, ASSIGN because future has completed 2024-11-25T03:15:59,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:15:59,795 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,795 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:15:59,796 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,796 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:15:59,796 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,796 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,797 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,798 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:59,798 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d74b21d2d7105b5e1e755662f34cb2a2 columnFamilyName A 2024-11-25T03:15:59,798 DEBUG [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:59,799 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.HStore(327): Store=d74b21d2d7105b5e1e755662f34cb2a2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:59,799 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,799 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:59,800 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d74b21d2d7105b5e1e755662f34cb2a2 columnFamilyName B 2024-11-25T03:15:59,800 DEBUG [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:59,800 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.HStore(327): Store=d74b21d2d7105b5e1e755662f34cb2a2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:59,800 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,801 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:15:59,801 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d74b21d2d7105b5e1e755662f34cb2a2 columnFamilyName C 2024-11-25T03:15:59,801 DEBUG [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:15:59,801 INFO [StoreOpener-d74b21d2d7105b5e1e755662f34cb2a2-1 {}] regionserver.HStore(327): Store=d74b21d2d7105b5e1e755662f34cb2a2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:15:59,801 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,801 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,802 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,802 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,803 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,803 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,803 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:15:59,804 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,806 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:15:59,806 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened d74b21d2d7105b5e1e755662f34cb2a2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59550869, jitterRate=-0.11262290179729462}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:15:59,806 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:15:59,806 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for d74b21d2d7105b5e1e755662f34cb2a2: Running coprocessor pre-open hook at 1732504559796Writing region info on filesystem at 1732504559796Initializing all the Stores at 1732504559796Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504559797 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504559797Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504559797Cleaning up temporary data from old regions at 1732504559803 (+6 ms)Running coprocessor post-open hooks at 1732504559806 (+3 ms)Region opened successfully at 1732504559806 2024-11-25T03:15:59,807 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., pid=126, masterSystemTime=1732504559792 2024-11-25T03:15:59,809 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,809 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:15:59,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=d74b21d2d7105b5e1e755662f34cb2a2, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:15:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-25T03:15:59,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:15:59,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-25T03:15:59,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 in 170 msec 2024-11-25T03:15:59,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-25T03:15:59,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, ASSIGN in 325 msec 2024-11-25T03:15:59,814 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:15:59,814 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504559814"}]},"ts":"1732504559814"} 2024-11-25T03:15:59,815 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:15:59,816 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:15:59,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-25T03:16:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-25T03:16:00,821 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:00,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,823 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306e46df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,824 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a43722, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,825 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,827 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:16:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,44623,1732504424313 2024-11-25T03:16:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46f5a04c 2024-11-25T03:16:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:16:00,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:16:00,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29de6219, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a0cfe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,833 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6652b3fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48cadd42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e14643a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,836 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49023761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,838 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73109920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,839 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5ecf95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,840 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbb3560, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,841 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34df56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,843 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcb4179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,844 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64766cbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,846 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c719ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5896992d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178e4d02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,849 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4673bee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,851 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0dd139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,852 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1548731c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:00,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:00,853 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32451be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:00,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e8d508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:00,859 DEBUG [Thread-1911 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,860 DEBUG [Thread-1926 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,860 DEBUG [Thread-1915 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,860 DEBUG [Thread-1922 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,860 DEBUG [Thread-1913 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:00,860 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,861 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,861 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,861 DEBUG [Thread-1924 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-25T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,862 DEBUG [Thread-1917 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:16:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:00,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,867 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2] 2024-11-25T03:16:00,869 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:00,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:00,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:16:00,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:00,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:00,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:00,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:00,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:00,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:00,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/f528538b4a3740bb96df9faa145b1e2c is 50, key is test_row_0/A:col10/1732504560868/Put/seqid=0 2024-11-25T03:16:00,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742246_1422 (size=16681) 2024-11-25T03:16:00,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504620924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:00,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504620924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504620926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504620926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:00,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504620938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:16:01,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504621050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504621050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504621050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504621051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504621051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:16:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504621270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504621270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504621270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504621270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504621270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/f528538b4a3740bb96df9faa145b1e2c 2024-11-25T03:16:01,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/2c3fa3340f2a4629acaa4d6325b4bdfe is 50, key is test_row_0/B:col10/1732504560868/Put/seqid=0 2024-11-25T03:16:01,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742247_1423 (size=12001) 2024-11-25T03:16:01,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:01,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:01,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:16:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504621580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504621580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504621580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504621580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504621580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:01,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/2c3fa3340f2a4629acaa4d6325b4bdfe 2024-11-25T03:16:01,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/06f94fb0b5f24011b850ad64414ec877 is 50, key is test_row_0/C:col10/1732504560868/Put/seqid=0 2024-11-25T03:16:01,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742248_1424 (size=12001) 2024-11-25T03:16:01,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:01,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:01,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:01,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:16:02,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:02,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:02,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:02,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504622090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:02,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504622090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504622100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504622100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504622100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:02,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/06f94fb0b5f24011b850ad64414ec877 2024-11-25T03:16:02,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/f528538b4a3740bb96df9faa145b1e2c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f528538b4a3740bb96df9faa145b1e2c 2024-11-25T03:16:02,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f528538b4a3740bb96df9faa145b1e2c, entries=250, sequenceid=14, filesize=16.3 K 2024-11-25T03:16:02,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/2c3fa3340f2a4629acaa4d6325b4bdfe as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/2c3fa3340f2a4629acaa4d6325b4bdfe 2024-11-25T03:16:02,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/2c3fa3340f2a4629acaa4d6325b4bdfe, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T03:16:02,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/06f94fb0b5f24011b850ad64414ec877 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/06f94fb0b5f24011b850ad64414ec877 2024-11-25T03:16:02,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/06f94fb0b5f24011b850ad64414ec877, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T03:16:02,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d74b21d2d7105b5e1e755662f34cb2a2 in 1303ms, sequenceid=14, compaction requested=false 2024-11-25T03:16:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:02,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:16:02,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:02,248 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:16:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:02,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/28a0a7867b314002ae87755cc3b023af is 50, key is test_row_0/A:col10/1732504560893/Put/seqid=0 2024-11-25T03:16:02,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742249_1425 (size=12001) 2024-11-25T03:16:02,659 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/28a0a7867b314002ae87755cc3b023af 2024-11-25T03:16:02,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5ee149934c3d426a87351442bb2bf933 is 50, key is test_row_0/B:col10/1732504560893/Put/seqid=0 2024-11-25T03:16:02,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742250_1426 (size=12001) 2024-11-25T03:16:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:16:03,072 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5ee149934c3d426a87351442bb2bf933 2024-11-25T03:16:03,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/c199688239df426dae54e97c709b8c51 is 50, key is test_row_0/C:col10/1732504560893/Put/seqid=0 2024-11-25T03:16:03,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742251_1427 (size=12001) 2024-11-25T03:16:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:03,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:03,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504623127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504623128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504623129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504623131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504623132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504623240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504623250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504623250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504623250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504623250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504623460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504623470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504623470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504623471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504623471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,484 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/c199688239df426dae54e97c709b8c51 2024-11-25T03:16:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/28a0a7867b314002ae87755cc3b023af as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/28a0a7867b314002ae87755cc3b023af 2024-11-25T03:16:03,491 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/28a0a7867b314002ae87755cc3b023af, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T03:16:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5ee149934c3d426a87351442bb2bf933 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5ee149934c3d426a87351442bb2bf933 2024-11-25T03:16:03,495 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5ee149934c3d426a87351442bb2bf933, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T03:16:03,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/c199688239df426dae54e97c709b8c51 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c199688239df426dae54e97c709b8c51 2024-11-25T03:16:03,498 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c199688239df426dae54e97c709b8c51, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T03:16:03,499 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d74b21d2d7105b5e1e755662f34cb2a2 in 1251ms, sequenceid=33, compaction requested=false 2024-11-25T03:16:03,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:03,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:03,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-25T03:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-25T03:16:03,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-25T03:16:03,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-25T03:16:03,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6430 sec 2024-11-25T03:16:03,756 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:16:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:03,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T03:16:03,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:03,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:03,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:03,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:03,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:03,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:03,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/5e9bedbfe70c4f3899b743ee2dafd435 is 50, key is test_row_0/A:col10/1732504563125/Put/seqid=0 2024-11-25T03:16:03,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742252_1428 (size=12001) 2024-11-25T03:16:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504623809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504623810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504623811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504623812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504623816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504623930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504623930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504623930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504623930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:03,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504623940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504624150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504624150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504624150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504624150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504624160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/5e9bedbfe70c4f3899b743ee2dafd435 2024-11-25T03:16:04,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/9d0965a505ea4cd283f97b76d0e3762c is 50, key is test_row_0/B:col10/1732504563125/Put/seqid=0 2024-11-25T03:16:04,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742253_1429 (size=12001) 2024-11-25T03:16:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504624460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504624460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504624460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504624460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504624480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/9d0965a505ea4cd283f97b76d0e3762c 2024-11-25T03:16:04,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/389e5c23acd041599df719e4d4d0c6b8 is 50, key is test_row_0/C:col10/1732504563125/Put/seqid=0 2024-11-25T03:16:04,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742254_1430 (size=12001) 2024-11-25T03:16:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504624980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504624980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504624980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504624980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:05,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504624990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:05,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/389e5c23acd041599df719e4d4d0c6b8 2024-11-25T03:16:05,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/5e9bedbfe70c4f3899b743ee2dafd435 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/5e9bedbfe70c4f3899b743ee2dafd435 2024-11-25T03:16:05,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/5e9bedbfe70c4f3899b743ee2dafd435, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T03:16:05,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/9d0965a505ea4cd283f97b76d0e3762c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9d0965a505ea4cd283f97b76d0e3762c 2024-11-25T03:16:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:16:05,031 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:05,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-25T03:16:05,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9d0965a505ea4cd283f97b76d0e3762c, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T03:16:05,034 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:05,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/389e5c23acd041599df719e4d4d0c6b8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/389e5c23acd041599df719e4d4d0c6b8 2024-11-25T03:16:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:16:05,034 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:05,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:05,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/389e5c23acd041599df719e4d4d0c6b8, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T03:16:05,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d74b21d2d7105b5e1e755662f34cb2a2 in 1254ms, sequenceid=47, compaction requested=true 2024-11-25T03:16:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:05,039 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:05,039 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:05,040 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:05,040 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:05,040 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:05,040 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:05,040 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:05,040 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:05,040 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f528538b4a3740bb96df9faa145b1e2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/28a0a7867b314002ae87755cc3b023af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/5e9bedbfe70c4f3899b743ee2dafd435] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=39.7 K 2024-11-25T03:16:05,040 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/2c3fa3340f2a4629acaa4d6325b4bdfe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5ee149934c3d426a87351442bb2bf933, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9d0965a505ea4cd283f97b76d0e3762c] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=35.2 K 2024-11-25T03:16:05,041 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3fa3340f2a4629acaa4d6325b4bdfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732504560868 2024-11-25T03:16:05,041 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f528538b4a3740bb96df9faa145b1e2c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732504560868 2024-11-25T03:16:05,041 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee149934c3d426a87351442bb2bf933, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732504560893 2024-11-25T03:16:05,041 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a0a7867b314002ae87755cc3b023af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732504560893 2024-11-25T03:16:05,041 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0965a505ea4cd283f97b76d0e3762c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732504563125 2024-11-25T03:16:05,041 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9bedbfe70c4f3899b743ee2dafd435, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732504563125 2024-11-25T03:16:05,049 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:05,049 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:05,050 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/0a1961fcc6264adfbd31a8d44955488f is 50, key is test_row_0/B:col10/1732504563125/Put/seqid=0 2024-11-25T03:16:05,050 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/1aac8d4cd4224822a21dc719f9d299b5 is 50, key is test_row_0/A:col10/1732504563125/Put/seqid=0 2024-11-25T03:16:05,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742256_1432 (size=12104) 2024-11-25T03:16:05,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742255_1431 (size=12104) 2024-11-25T03:16:05,064 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/1aac8d4cd4224822a21dc719f9d299b5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/1aac8d4cd4224822a21dc719f9d299b5 2024-11-25T03:16:05,068 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into 1aac8d4cd4224822a21dc719f9d299b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:05,068 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:05,068 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=13, startTime=1732504565038; duration=0sec 2024-11-25T03:16:05,068 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:05,068 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:05,068 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:05,069 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:05,069 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:05,069 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:05,069 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/06f94fb0b5f24011b850ad64414ec877, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c199688239df426dae54e97c709b8c51, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/389e5c23acd041599df719e4d4d0c6b8] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=35.2 K 2024-11-25T03:16:05,070 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f94fb0b5f24011b850ad64414ec877, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732504560868 2024-11-25T03:16:05,070 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c199688239df426dae54e97c709b8c51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732504560893 2024-11-25T03:16:05,073 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389e5c23acd041599df719e4d4d0c6b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732504563125 2024-11-25T03:16:05,078 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:05,078 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/15c1c2e4682d4525a8d39e689bc39a86 is 50, key is test_row_0/C:col10/1732504563125/Put/seqid=0 2024-11-25T03:16:05,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742257_1433 (size=12104) 2024-11-25T03:16:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:16:05,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-25T03:16:05,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:05,187 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:16:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:05,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/cb78ac46209644899253d3625cf1ddbb is 50, key is test_row_0/A:col10/1732504563811/Put/seqid=0 2024-11-25T03:16:05,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742258_1434 (size=12001) 2024-11-25T03:16:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:16:05,463 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/0a1961fcc6264adfbd31a8d44955488f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/0a1961fcc6264adfbd31a8d44955488f 2024-11-25T03:16:05,468 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into 0a1961fcc6264adfbd31a8d44955488f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:05,468 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:05,468 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=13, startTime=1732504565039; duration=0sec 2024-11-25T03:16:05,468 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:05,468 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:05,486 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/15c1c2e4682d4525a8d39e689bc39a86 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/15c1c2e4682d4525a8d39e689bc39a86 2024-11-25T03:16:05,490 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into 15c1c2e4682d4525a8d39e689bc39a86(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:05,490 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:05,490 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=13, startTime=1732504565039; duration=0sec 2024-11-25T03:16:05,490 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:05,490 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:05,597 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/cb78ac46209644899253d3625cf1ddbb 2024-11-25T03:16:05,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/978ec1cdd0124922945c81a37660c19f is 50, key is test_row_0/B:col10/1732504563811/Put/seqid=0 2024-11-25T03:16:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742259_1435 (size=12001) 2024-11-25T03:16:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:16:06,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:06,008 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/978ec1cdd0124922945c81a37660c19f 2024-11-25T03:16:06,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/00fda2a08a82447d8a69f76fea7157c7 is 50, key is test_row_0/C:col10/1732504563811/Put/seqid=0 2024-11-25T03:16:06,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742260_1436 (size=12001) 2024-11-25T03:16:06,020 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/00fda2a08a82447d8a69f76fea7157c7 2024-11-25T03:16:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/cb78ac46209644899253d3625cf1ddbb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/cb78ac46209644899253d3625cf1ddbb 2024-11-25T03:16:06,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504626017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504626019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,027 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/cb78ac46209644899253d3625cf1ddbb, entries=150, sequenceid=66, filesize=11.7 K 2024-11-25T03:16:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/978ec1cdd0124922945c81a37660c19f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/978ec1cdd0124922945c81a37660c19f 2024-11-25T03:16:06,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,031 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/978ec1cdd0124922945c81a37660c19f, entries=150, sequenceid=66, filesize=11.7 K 2024-11-25T03:16:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504626021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504626022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/00fda2a08a82447d8a69f76fea7157c7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/00fda2a08a82447d8a69f76fea7157c7 2024-11-25T03:16:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,035 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/00fda2a08a82447d8a69f76fea7157c7, entries=150, sequenceid=66, filesize=11.7 K 2024-11-25T03:16:06,036 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d74b21d2d7105b5e1e755662f34cb2a2 in 849ms, sequenceid=66, compaction requested=false 2024-11-25T03:16:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-25T03:16:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:06,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:16:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-25T03:16:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:06,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:06,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:06,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:06,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:06,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-25T03:16:06,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-25T03:16:06,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/80b4bcb442314817899478b23272d90e is 50, key is test_row_0/A:col10/1732504566035/Put/seqid=0 2024-11-25T03:16:06,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0120 sec 2024-11-25T03:16:06,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742261_1437 (size=14341) 2024-11-25T03:16:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504626130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504626133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504626140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504626140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504626141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:16:06,170 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:06,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-25T03:16:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:16:06,173 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:06,173 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:06,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:06,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504626250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504626250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:16:06,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504626350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504626350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504626350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/80b4bcb442314817899478b23272d90e 2024-11-25T03:16:06,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/4586565eb9a642cfb286cc15b4853f28 is 50, key is test_row_0/B:col10/1732504566035/Put/seqid=0 2024-11-25T03:16:06,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742262_1438 (size=12001) 2024-11-25T03:16:06,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:06,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504626470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504626470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:16:06,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504626670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504626670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504626670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504626790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504626790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:16:06,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/4586565eb9a642cfb286cc15b4853f28 2024-11-25T03:16:06,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/ec6c00be327341b8a3d36b8c8d49dfa2 is 50, key is test_row_0/C:col10/1732504566035/Put/seqid=0 2024-11-25T03:16:06,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742263_1439 (size=12001) 2024-11-25T03:16:06,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:06,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504627189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504627190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504627190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/ec6c00be327341b8a3d36b8c8d49dfa2 2024-11-25T03:16:07,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/80b4bcb442314817899478b23272d90e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/80b4bcb442314817899478b23272d90e 2024-11-25T03:16:07,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/80b4bcb442314817899478b23272d90e, entries=200, sequenceid=83, filesize=14.0 K 2024-11-25T03:16:07,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/4586565eb9a642cfb286cc15b4853f28 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4586565eb9a642cfb286cc15b4853f28 2024-11-25T03:16:07,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4586565eb9a642cfb286cc15b4853f28, entries=150, sequenceid=83, filesize=11.7 K 2024-11-25T03:16:07,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/ec6c00be327341b8a3d36b8c8d49dfa2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ec6c00be327341b8a3d36b8c8d49dfa2 2024-11-25T03:16:07,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ec6c00be327341b8a3d36b8c8d49dfa2, entries=150, sequenceid=83, filesize=11.7 K 2024-11-25T03:16:07,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d74b21d2d7105b5e1e755662f34cb2a2 in 1249ms, sequenceid=83, compaction requested=true 2024-11-25T03:16:07,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:07,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:07,286 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:07,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:07,287 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:07,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:07,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:07,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:07,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:07,287 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:07,287 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:07,287 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:07,287 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:07,287 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,287 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,288 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/0a1961fcc6264adfbd31a8d44955488f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/978ec1cdd0124922945c81a37660c19f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4586565eb9a642cfb286cc15b4853f28] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=35.3 K 2024-11-25T03:16:07,288 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/1aac8d4cd4224822a21dc719f9d299b5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/cb78ac46209644899253d3625cf1ddbb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/80b4bcb442314817899478b23272d90e] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=37.5 K 2024-11-25T03:16:07,288 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aac8d4cd4224822a21dc719f9d299b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732504563125 2024-11-25T03:16:07,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a1961fcc6264adfbd31a8d44955488f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732504563125 2024-11-25T03:16:07,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 978ec1cdd0124922945c81a37660c19f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732504563810 2024-11-25T03:16:07,288 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb78ac46209644899253d3625cf1ddbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732504563810 2024-11-25T03:16:07,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4586565eb9a642cfb286cc15b4853f28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732504566004 2024-11-25T03:16:07,289 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b4bcb442314817899478b23272d90e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732504566004 2024-11-25T03:16:07,295 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:07,295 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:07,295 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/923e79e503c54654bb0ae1c105d335ce is 50, key is test_row_0/B:col10/1732504566035/Put/seqid=0 2024-11-25T03:16:07,295 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/15c76ede6ac0444e89d1944ced0fdc90 is 50, key is test_row_0/A:col10/1732504566035/Put/seqid=0 2024-11-25T03:16:07,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742265_1441 (size=12207) 2024-11-25T03:16:07,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742264_1440 (size=12207) 2024-11-25T03:16:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:16:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:07,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:16:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:07,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/c22260762ef443c291f43119c44c49ce is 50, key is test_row_0/A:col10/1732504567315/Put/seqid=0 2024-11-25T03:16:07,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742266_1442 (size=16681) 2024-11-25T03:16:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504627353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504627359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:07,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:07,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504627470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504627479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:07,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:07,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504627690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504627690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:07,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:07,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:07,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,705 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/923e79e503c54654bb0ae1c105d335ce as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/923e79e503c54654bb0ae1c105d335ce 2024-11-25T03:16:07,710 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into 923e79e503c54654bb0ae1c105d335ce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:07,710 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:07,710 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=13, startTime=1732504567287; duration=0sec 2024-11-25T03:16:07,710 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:07,710 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:07,711 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:07,711 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/15c76ede6ac0444e89d1944ced0fdc90 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/15c76ede6ac0444e89d1944ced0fdc90 2024-11-25T03:16:07,711 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:07,712 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:07,712 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,712 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/15c1c2e4682d4525a8d39e689bc39a86, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/00fda2a08a82447d8a69f76fea7157c7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ec6c00be327341b8a3d36b8c8d49dfa2] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=35.3 K 2024-11-25T03:16:07,712 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c1c2e4682d4525a8d39e689bc39a86, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732504563125 2024-11-25T03:16:07,713 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 00fda2a08a82447d8a69f76fea7157c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732504563810 2024-11-25T03:16:07,713 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ec6c00be327341b8a3d36b8c8d49dfa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732504566004 2024-11-25T03:16:07,715 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into 15c76ede6ac0444e89d1944ced0fdc90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:07,715 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:07,715 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=13, startTime=1732504567286; duration=0sec 2024-11-25T03:16:07,716 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:07,716 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:07,725 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:07,725 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/a12b6dd451a0466b8a72107c04b4e9b4 is 50, key is test_row_0/C:col10/1732504566035/Put/seqid=0 2024-11-25T03:16:07,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/c22260762ef443c291f43119c44c49ce 2024-11-25T03:16:07,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742267_1443 (size=12207) 2024-11-25T03:16:07,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/9ddc49d4f5314c09afb524ebda8ac6b8 is 50, key is test_row_0/B:col10/1732504567315/Put/seqid=0 2024-11-25T03:16:07,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742268_1444 (size=12001) 2024-11-25T03:16:07,737 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/a12b6dd451a0466b8a72107c04b4e9b4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/a12b6dd451a0466b8a72107c04b4e9b4 2024-11-25T03:16:07,742 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into a12b6dd451a0466b8a72107c04b4e9b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:07,742 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:07,742 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=13, startTime=1732504567287; duration=0sec 2024-11-25T03:16:07,742 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:07,742 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:07,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:07,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:07,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504628010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:08,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504628010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:08,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/9ddc49d4f5314c09afb524ebda8ac6b8 2024-11-25T03:16:08,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/43e8c57a06ec44cf95aa367dd7f86ae0 is 50, key is test_row_0/C:col10/1732504567315/Put/seqid=0 2024-11-25T03:16:08,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742269_1445 (size=12001) 2024-11-25T03:16:08,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504628210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504628210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504628210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:08,218 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:08,218 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:08,219 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:08,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:16:08,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:08,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:08,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504628520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504628521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:08,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/43e8c57a06ec44cf95aa367dd7f86ae0 2024-11-25T03:16:08,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/c22260762ef443c291f43119c44c49ce as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c22260762ef443c291f43119c44c49ce 2024-11-25T03:16:08,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c22260762ef443c291f43119c44c49ce, entries=250, sequenceid=102, filesize=16.3 K 2024-11-25T03:16:08,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/9ddc49d4f5314c09afb524ebda8ac6b8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9ddc49d4f5314c09afb524ebda8ac6b8 2024-11-25T03:16:08,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9ddc49d4f5314c09afb524ebda8ac6b8, entries=150, sequenceid=102, filesize=11.7 K 2024-11-25T03:16:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/43e8c57a06ec44cf95aa367dd7f86ae0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/43e8c57a06ec44cf95aa367dd7f86ae0 2024-11-25T03:16:08,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/43e8c57a06ec44cf95aa367dd7f86ae0, entries=150, sequenceid=102, filesize=11.7 K 2024-11-25T03:16:08,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d74b21d2d7105b5e1e755662f34cb2a2 in 1248ms, sequenceid=102, compaction requested=false 2024-11-25T03:16:08,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:08,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:16:08,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:08,623 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T03:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:08,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/6312eb09b7e04f628ff8f73227eebf50 is 50, key is test_row_0/A:col10/1732504567358/Put/seqid=0 2024-11-25T03:16:08,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742270_1446 (size=12001) 2024-11-25T03:16:09,033 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/6312eb09b7e04f628ff8f73227eebf50 2024-11-25T03:16:09,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/af9ac7b8b0eb43e093c1161911457539 is 50, key is test_row_0/B:col10/1732504567358/Put/seqid=0 2024-11-25T03:16:09,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742271_1447 (size=12001) 2024-11-25T03:16:09,445 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/af9ac7b8b0eb43e093c1161911457539 2024-11-25T03:16:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/809a656d760340b49fcb336f2c5f148a is 50, key is test_row_0/C:col10/1732504567358/Put/seqid=0 2024-11-25T03:16:09,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742272_1448 (size=12001) 2024-11-25T03:16:09,462 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/809a656d760340b49fcb336f2c5f148a 2024-11-25T03:16:09,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/6312eb09b7e04f628ff8f73227eebf50 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6312eb09b7e04f628ff8f73227eebf50 2024-11-25T03:16:09,474 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6312eb09b7e04f628ff8f73227eebf50, entries=150, sequenceid=118, filesize=11.7 K 2024-11-25T03:16:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/af9ac7b8b0eb43e093c1161911457539 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/af9ac7b8b0eb43e093c1161911457539 2024-11-25T03:16:09,479 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/af9ac7b8b0eb43e093c1161911457539, entries=150, sequenceid=118, filesize=11.7 K 2024-11-25T03:16:09,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/809a656d760340b49fcb336f2c5f148a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/809a656d760340b49fcb336f2c5f148a 2024-11-25T03:16:09,485 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/809a656d760340b49fcb336f2c5f148a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-25T03:16:09,488 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for d74b21d2d7105b5e1e755662f34cb2a2 in 865ms, sequenceid=118, compaction requested=true 2024-11-25T03:16:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-25T03:16:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-25T03:16:09,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-25T03:16:09,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3170 sec 2024-11-25T03:16:09,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 3.3220 sec 2024-11-25T03:16:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:09,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:16:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:09,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/9a900485f66c4304bc12a3cd6e88d924 is 50, key is test_row_0/A:col10/1732504569553/Put/seqid=0 2024-11-25T03:16:09,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742273_1449 (size=14341) 2024-11-25T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504569625, max timestamp=1732504569650Num uniques 859; 2024-11-25T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504569625, max timestamp=1732504569650Num uniques 859; 2024-11-25T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504569625, max timestamp=1732504569650Num uniques 859; 2024-11-25T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:09,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504629748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:09,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504629759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:09,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504629870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:09,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504629870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:09,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:09,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/9a900485f66c4304bc12a3cd6e88d924 2024-11-25T03:16:09,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ebc857bb25674cd295c9cf4678ae3bba is 50, key is test_row_0/B:col10/1732504569553/Put/seqid=0 2024-11-25T03:16:09,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742274_1450 (size=12001) 2024-11-25T03:16:10,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504630090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504630090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504630239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504630239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,253 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:10,253 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504630250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,261 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:16:10,330 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:10,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-25T03:16:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:16:10,332 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:10,333 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:10,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:10,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ebc857bb25674cd295c9cf4678ae3bba 2024-11-25T03:16:10,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/08273bf5bdfa4e80a6b9b112e9576399 is 50, key is test_row_0/C:col10/1732504569553/Put/seqid=0 2024-11-25T03:16:10,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742275_1451 (size=12001) 2024-11-25T03:16:10,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/08273bf5bdfa4e80a6b9b112e9576399 2024-11-25T03:16:10,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/9a900485f66c4304bc12a3cd6e88d924 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a900485f66c4304bc12a3cd6e88d924 2024-11-25T03:16:10,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a900485f66c4304bc12a3cd6e88d924, entries=200, sequenceid=128, filesize=14.0 K 2024-11-25T03:16:10,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ebc857bb25674cd295c9cf4678ae3bba as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ebc857bb25674cd295c9cf4678ae3bba 2024-11-25T03:16:10,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ebc857bb25674cd295c9cf4678ae3bba, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:16:10,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/08273bf5bdfa4e80a6b9b112e9576399 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/08273bf5bdfa4e80a6b9b112e9576399 2024-11-25T03:16:10,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/08273bf5bdfa4e80a6b9b112e9576399, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:16:10,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d74b21d2d7105b5e1e755662f34cb2a2 in 848ms, sequenceid=128, compaction requested=true 2024-11-25T03:16:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:10,403 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:16:10,403 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:16:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:10,404 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55230 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:16:10,404 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48210 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:16:10,404 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:10,404 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:10,404 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,404 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,404 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/15c76ede6ac0444e89d1944ced0fdc90, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c22260762ef443c291f43119c44c49ce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6312eb09b7e04f628ff8f73227eebf50, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a900485f66c4304bc12a3cd6e88d924] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=53.9 K 2024-11-25T03:16:10,404 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/923e79e503c54654bb0ae1c105d335ce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9ddc49d4f5314c09afb524ebda8ac6b8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/af9ac7b8b0eb43e093c1161911457539, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ebc857bb25674cd295c9cf4678ae3bba] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=47.1 K 2024-11-25T03:16:10,404 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c76ede6ac0444e89d1944ced0fdc90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732504566004 2024-11-25T03:16:10,404 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 923e79e503c54654bb0ae1c105d335ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732504566004 2024-11-25T03:16:10,405 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22260762ef443c291f43119c44c49ce, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732504566064 2024-11-25T03:16:10,405 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ddc49d4f5314c09afb524ebda8ac6b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732504566064 2024-11-25T03:16:10,405 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6312eb09b7e04f628ff8f73227eebf50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732504567350 2024-11-25T03:16:10,405 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting af9ac7b8b0eb43e093c1161911457539, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732504567350 2024-11-25T03:16:10,405 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a900485f66c4304bc12a3cd6e88d924, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504569543 2024-11-25T03:16:10,405 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ebc857bb25674cd295c9cf4678ae3bba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504569547 2024-11-25T03:16:10,413 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:10,414 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ac3f503661474a74815875c9e362b8b7 is 50, key is test_row_0/B:col10/1732504569553/Put/seqid=0 2024-11-25T03:16:10,414 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:10,415 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/7a316d47e55a4b96963c076521043651 is 50, key is test_row_0/A:col10/1732504569553/Put/seqid=0 2024-11-25T03:16:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:10,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-25T03:16:10,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:10,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:10,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:10,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:10,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:10,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:10,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742276_1452 (size=12343) 2024-11-25T03:16:10,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/14c456f6ef7c493a94a94c7d4c7c59cc is 50, key is test_row_0/A:col10/1732504569741/Put/seqid=0 2024-11-25T03:16:10,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742277_1453 (size=12343) 2024-11-25T03:16:10,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742278_1454 (size=16931) 2024-11-25T03:16:10,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/14c456f6ef7c493a94a94c7d4c7c59cc 2024-11-25T03:16:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:16:10,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/a5021b3eddff4acf8eae5645c11263d5 is 50, key is test_row_0/B:col10/1732504569741/Put/seqid=0 2024-11-25T03:16:10,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742279_1455 (size=12151) 2024-11-25T03:16:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504630452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504630472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:16:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504630580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504630580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:16:10,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:10,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:16:10,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:16:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504630800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504630800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:10,834 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ac3f503661474a74815875c9e362b8b7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ac3f503661474a74815875c9e362b8b7 2024-11-25T03:16:10,839 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into ac3f503661474a74815875c9e362b8b7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:10,839 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:10,839 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=12, startTime=1732504570403; duration=0sec 2024-11-25T03:16:10,839 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:10,839 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:10,839 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:16:10,841 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48210 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:16:10,841 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:10,841 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,841 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/a12b6dd451a0466b8a72107c04b4e9b4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/43e8c57a06ec44cf95aa367dd7f86ae0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/809a656d760340b49fcb336f2c5f148a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/08273bf5bdfa4e80a6b9b112e9576399] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=47.1 K 2024-11-25T03:16:10,842 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a12b6dd451a0466b8a72107c04b4e9b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732504566004 2024-11-25T03:16:10,842 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/7a316d47e55a4b96963c076521043651 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7a316d47e55a4b96963c076521043651 2024-11-25T03:16:10,842 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e8c57a06ec44cf95aa367dd7f86ae0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732504566064 2024-11-25T03:16:10,843 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 809a656d760340b49fcb336f2c5f148a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732504567350 2024-11-25T03:16:10,843 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 08273bf5bdfa4e80a6b9b112e9576399, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504569547 2024-11-25T03:16:10,847 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into 7a316d47e55a4b96963c076521043651(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:10,847 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:10,847 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=12, startTime=1732504570403; duration=0sec 2024-11-25T03:16:10,847 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:10,847 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:10,852 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:10,853 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/ce5f9f339e704d0ebea72d911146b212 is 50, key is test_row_0/C:col10/1732504569553/Put/seqid=0 2024-11-25T03:16:10,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/a5021b3eddff4acf8eae5645c11263d5 2024-11-25T03:16:10,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742280_1456 (size=12343) 2024-11-25T03:16:10,862 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/ce5f9f339e704d0ebea72d911146b212 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ce5f9f339e704d0ebea72d911146b212 2024-11-25T03:16:10,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/f6332c6a4c1e4e648eea6ed40d62e0a5 is 50, key is test_row_0/C:col10/1732504569741/Put/seqid=0 2024-11-25T03:16:10,866 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into ce5f9f339e704d0ebea72d911146b212(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:10,866 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:10,866 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=12, startTime=1732504570403; duration=0sec 2024-11-25T03:16:10,866 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:10,866 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:10,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742281_1457 (size=12151) 2024-11-25T03:16:10,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/f6332c6a4c1e4e648eea6ed40d62e0a5 2024-11-25T03:16:10,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/14c456f6ef7c493a94a94c7d4c7c59cc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/14c456f6ef7c493a94a94c7d4c7c59cc 2024-11-25T03:16:10,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/14c456f6ef7c493a94a94c7d4c7c59cc, entries=250, sequenceid=167, filesize=16.5 K 2024-11-25T03:16:10,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/a5021b3eddff4acf8eae5645c11263d5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a5021b3eddff4acf8eae5645c11263d5 2024-11-25T03:16:10,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a5021b3eddff4acf8eae5645c11263d5, entries=150, sequenceid=167, filesize=11.9 K 2024-11-25T03:16:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/f6332c6a4c1e4e648eea6ed40d62e0a5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/f6332c6a4c1e4e648eea6ed40d62e0a5 2024-11-25T03:16:10,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/f6332c6a4c1e4e648eea6ed40d62e0a5, entries=150, sequenceid=167, filesize=11.9 K 2024-11-25T03:16:10,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for d74b21d2d7105b5e1e755662f34cb2a2 in 473ms, sequenceid=167, compaction requested=false 2024-11-25T03:16:10,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:10,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:16:10,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:10,945 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T03:16:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:10,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/3c2340a5b4e24d2ab8cd459f8f315b78 is 50, key is test_row_0/A:col10/1732504570450/Put/seqid=0 2024-11-25T03:16:10,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742282_1458 (size=12151) 2024-11-25T03:16:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:16:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:11,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504571155, max timestamp=1732504571193Num uniques 859; 2024-11-25T03:16:11,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504571155, max timestamp=1732504571193Num uniques 859; 2024-11-25T03:16:11,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:16:11,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504571155, max timestamp=1732504571193Num uniques 859; 2024-11-25T03:16:11,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:16:11,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:16:11,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504571235, max timestamp=1732504571277Num uniques 1068; 2024-11-25T03:16:11,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504571235, max timestamp=1732504571277Num uniques 1068; 2024-11-25T03:16:11,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-25T03:16:11,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-25T03:16:11,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504571235, max timestamp=1732504571277Num uniques 1068; 2024-11-25T03:16:11,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-25T03:16:11,356 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/3c2340a5b4e24d2ab8cd459f8f315b78 2024-11-25T03:16:11,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/15250359885e4af8ac3c45cf3605f19d is 50, key is test_row_0/B:col10/1732504570450/Put/seqid=0 2024-11-25T03:16:11,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742283_1459 (size=12151) 2024-11-25T03:16:11,371 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/15250359885e4af8ac3c45cf3605f19d 2024-11-25T03:16:11,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732504571352, max timestamp=1732504571373Num uniques 1077; 2024-11-25T03:16:11,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-25T03:16:11,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732504571352, max timestamp=1732504571373Num uniques 1077; 2024-11-25T03:16:11,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-25T03:16:11,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732504571352, max timestamp=1732504571373Num uniques 1077; 2024-11-25T03:16:11,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-25T03:16:11,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/aa48171bd82f49b1a35720ab1f40ca74 is 50, key is test_row_0/C:col10/1732504570450/Put/seqid=0 2024-11-25T03:16:11,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742284_1460 (size=12151) 2024-11-25T03:16:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:16:11,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732504571432, max timestamp=1732504571466Num uniques 1036; 2024-11-25T03:16:11,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-25T03:16:11,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732504571432, max timestamp=1732504571466Num uniques 1036; 2024-11-25T03:16:11,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-25T03:16:11,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732504571432, max timestamp=1732504571466Num uniques 1036; 2024-11-25T03:16:11,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-25T03:16:11,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504631567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:11,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732504571535, max timestamp=1732504571564Num uniques 1045; 2024-11-25T03:16:11,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-25T03:16:11,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732504571535, max timestamp=1732504571564Num uniques 1045; 2024-11-25T03:16:11,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-25T03:16:11,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732504571535, max timestamp=1732504571564Num uniques 1045; 2024-11-25T03:16:11,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-25T03:16:11,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732504571694, max timestamp=1732504571719Num uniques 1054; 2024-11-25T03:16:11,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732504571694, max timestamp=1732504571719Num uniques 1054; 2024-11-25T03:16:11,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-25T03:16:11,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-25T03:16:11,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732504571694, max timestamp=1732504571719Num uniques 1054; 2024-11-25T03:16:11,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-25T03:16:11,787 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/aa48171bd82f49b1a35720ab1f40ca74 2024-11-25T03:16:11,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/3c2340a5b4e24d2ab8cd459f8f315b78 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/3c2340a5b4e24d2ab8cd459f8f315b78 2024-11-25T03:16:11,794 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/3c2340a5b4e24d2ab8cd459f8f315b78, entries=150, sequenceid=178, filesize=11.9 K 2024-11-25T03:16:11,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/15250359885e4af8ac3c45cf3605f19d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/15250359885e4af8ac3c45cf3605f19d 2024-11-25T03:16:11,799 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/15250359885e4af8ac3c45cf3605f19d, entries=150, sequenceid=178, filesize=11.9 K 2024-11-25T03:16:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/aa48171bd82f49b1a35720ab1f40ca74 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/aa48171bd82f49b1a35720ab1f40ca74 2024-11-25T03:16:11,803 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/aa48171bd82f49b1a35720ab1f40ca74, entries=150, sequenceid=178, filesize=11.9 K 2024-11-25T03:16:11,804 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=127.47 KB/130530 for d74b21d2d7105b5e1e755662f34cb2a2 in 859ms, sequenceid=178, compaction requested=true 2024-11-25T03:16:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-25T03:16:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-25T03:16:11,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-25T03:16:11,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-25T03:16:11,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4780 sec 2024-11-25T03:16:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:11,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=134.18 KB heapSize=380.77 KB 2024-11-25T03:16:11,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:11,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:11,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:11,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:11,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:11,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:11,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/34c7fcb2deaf4991bdb431f643908686 is 50, key is test_row_0/A:col10/1732504571812/Put/seqid=0 2024-11-25T03:16:11,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742285_1461 (size=14741) 2024-11-25T03:16:11,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504631852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504631853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504631970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504631980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504632190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:12,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:12,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504632200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:12,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/34c7fcb2deaf4991bdb431f643908686 2024-11-25T03:16:12,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/d75892242cf04ac8a65e6de7aeefd189 is 50, key is test_row_0/B:col10/1732504571812/Put/seqid=0 2024-11-25T03:16:12,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742286_1462 (size=12301) 2024-11-25T03:16:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:16:12,480 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:12,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-25T03:16:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:16:12,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:12,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:12,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:12,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504632500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:12,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504632520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:16:12,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/d75892242cf04ac8a65e6de7aeefd189 2024-11-25T03:16:12,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-25T03:16:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:12,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/1eada155cc924cc299abc4392732575f is 50, key is test_row_0/C:col10/1732504571812/Put/seqid=0 2024-11-25T03:16:12,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742287_1463 (size=12301) 2024-11-25T03:16:12,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-25T03:16:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:12,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:16:12,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-25T03:16:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:12,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504633020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:13,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/1eada155cc924cc299abc4392732575f 2024-11-25T03:16:13,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/34c7fcb2deaf4991bdb431f643908686 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/34c7fcb2deaf4991bdb431f643908686 2024-11-25T03:16:13,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/34c7fcb2deaf4991bdb431f643908686, entries=200, sequenceid=300, filesize=14.4 K 2024-11-25T03:16:13,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/d75892242cf04ac8a65e6de7aeefd189 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d75892242cf04ac8a65e6de7aeefd189 2024-11-25T03:16:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504633040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:13,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d75892242cf04ac8a65e6de7aeefd189, entries=150, sequenceid=300, filesize=12.0 K 2024-11-25T03:16:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/1eada155cc924cc299abc4392732575f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/1eada155cc924cc299abc4392732575f 2024-11-25T03:16:13,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/1eada155cc924cc299abc4392732575f, entries=150, sequenceid=300, filesize=12.0 K 2024-11-25T03:16:13,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.72 KB/389856, currentSize=46.96 KB/48090 for d74b21d2d7105b5e1e755662f34cb2a2 in 1247ms, sequenceid=300, compaction requested=true 2024-11-25T03:16:13,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:13,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:13,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:13,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:13,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:13,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:13,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:13,060 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:16:13,060 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:16:13,061 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:16:13,061 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:16:13,061 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:13,061 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:13,061 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:13,061 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:13,061 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7a316d47e55a4b96963c076521043651, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/14c456f6ef7c493a94a94c7d4c7c59cc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/3c2340a5b4e24d2ab8cd459f8f315b78, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/34c7fcb2deaf4991bdb431f643908686] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=54.8 K 2024-11-25T03:16:13,061 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ac3f503661474a74815875c9e362b8b7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a5021b3eddff4acf8eae5645c11263d5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/15250359885e4af8ac3c45cf3605f19d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d75892242cf04ac8a65e6de7aeefd189] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=47.8 K 2024-11-25T03:16:13,062 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3f503661474a74815875c9e362b8b7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504569547 2024-11-25T03:16:13,062 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a316d47e55a4b96963c076521043651, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504569547 2024-11-25T03:16:13,062 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a5021b3eddff4acf8eae5645c11263d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732504569741 2024-11-25T03:16:13,062 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c456f6ef7c493a94a94c7d4c7c59cc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732504569741 2024-11-25T03:16:13,062 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 15250359885e4af8ac3c45cf3605f19d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732504570448 2024-11-25T03:16:13,062 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c2340a5b4e24d2ab8cd459f8f315b78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732504570448 2024-11-25T03:16:13,063 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d75892242cf04ac8a65e6de7aeefd189, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732504571764 2024-11-25T03:16:13,063 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c7fcb2deaf4991bdb431f643908686, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732504571764 2024-11-25T03:16:13,070 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:13,070 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:13,070 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/6feb4802efc54047b7fc67957bce22b4 is 50, key is test_row_0/B:col10/1732504571812/Put/seqid=0 2024-11-25T03:16:13,070 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/888d6c67fa00434b9266dab8d51c7dcb is 50, key is test_row_0/A:col10/1732504571812/Put/seqid=0 2024-11-25T03:16:13,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742289_1465 (size=12779) 2024-11-25T03:16:13,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742288_1464 (size=12779) 2024-11-25T03:16:13,083 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/888d6c67fa00434b9266dab8d51c7dcb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/888d6c67fa00434b9266dab8d51c7dcb 2024-11-25T03:16:13,086 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/6feb4802efc54047b7fc67957bce22b4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/6feb4802efc54047b7fc67957bce22b4 2024-11-25T03:16:13,090 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into 888d6c67fa00434b9266dab8d51c7dcb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:13,090 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:13,090 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=12, startTime=1732504573060; duration=0sec 2024-11-25T03:16:13,090 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:13,090 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:13,091 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:16:13,093 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:16:13,093 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:13,093 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:13,093 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ce5f9f339e704d0ebea72d911146b212, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/f6332c6a4c1e4e648eea6ed40d62e0a5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/aa48171bd82f49b1a35720ab1f40ca74, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/1eada155cc924cc299abc4392732575f] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=47.8 K 2024-11-25T03:16:13,093 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into 6feb4802efc54047b7fc67957bce22b4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:13,093 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:13,093 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=12, startTime=1732504573060; duration=0sec 2024-11-25T03:16:13,093 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:13,093 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:13,094 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5f9f339e704d0ebea72d911146b212, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732504569547 2024-11-25T03:16:13,094 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6332c6a4c1e4e648eea6ed40d62e0a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732504569741 2024-11-25T03:16:13,094 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa48171bd82f49b1a35720ab1f40ca74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732504570448 2024-11-25T03:16:13,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-25T03:16:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:13,095 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T03:16:13,095 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eada155cc924cc299abc4392732575f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732504571764 2024-11-25T03:16:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/a371aaac6d0741f591dc294039defbc1 is 50, key is test_row_0/A:col10/1732504571839/Put/seqid=0 2024-11-25T03:16:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742290_1466 (size=12301) 2024-11-25T03:16:13,108 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#396 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:13,108 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/adcc8e0d3105415696eeb12294bdad04 is 50, key is test_row_0/C:col10/1732504571812/Put/seqid=0 2024-11-25T03:16:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:16:13,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742291_1467 (size=12779) 2024-11-25T03:16:13,187 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:16:13,503 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/a371aaac6d0741f591dc294039defbc1 2024-11-25T03:16:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/294ef3d0bad5410ab8d2816cd5d63c6e is 50, key is test_row_0/B:col10/1732504571839/Put/seqid=0 2024-11-25T03:16:13,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742292_1468 (size=12301) 2024-11-25T03:16:13,522 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/adcc8e0d3105415696eeb12294bdad04 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/adcc8e0d3105415696eeb12294bdad04 2024-11-25T03:16:13,526 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into adcc8e0d3105415696eeb12294bdad04(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:13,526 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:13,526 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=12, startTime=1732504573060; duration=0sec 2024-11-25T03:16:13,526 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:13,526 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:16:13,914 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/294ef3d0bad5410ab8d2816cd5d63c6e 2024-11-25T03:16:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/eccd246a346641c5bdeff3a31c78983e is 50, key is test_row_0/C:col10/1732504571839/Put/seqid=0 2024-11-25T03:16:13,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742293_1469 (size=12301) 2024-11-25T03:16:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:14,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:14,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504574126, max timestamp=1732504574149Num uniques 859; 2024-11-25T03:16:14,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504574126, max timestamp=1732504574149Num uniques 859; 2024-11-25T03:16:14,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504574126, max timestamp=1732504574149Num uniques 859; 2024-11-25T03:16:14,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:14,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:14,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:14,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504634237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504634238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1732504634270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,280 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1732504634280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36310 deadline: 1732504634280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,292 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,292 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in d74b21d2d7105b5e1e755662f34cb2a2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,327 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/eccd246a346641c5bdeff3a31c78983e 2024-11-25T03:16:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/a371aaac6d0741f591dc294039defbc1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/a371aaac6d0741f591dc294039defbc1 2024-11-25T03:16:14,334 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/a371aaac6d0741f591dc294039defbc1, entries=150, sequenceid=312, filesize=12.0 K 2024-11-25T03:16:14,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/294ef3d0bad5410ab8d2816cd5d63c6e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/294ef3d0bad5410ab8d2816cd5d63c6e 2024-11-25T03:16:14,338 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/294ef3d0bad5410ab8d2816cd5d63c6e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-25T03:16:14,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/eccd246a346641c5bdeff3a31c78983e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/eccd246a346641c5bdeff3a31c78983e 2024-11-25T03:16:14,341 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/eccd246a346641c5bdeff3a31c78983e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-25T03:16:14,342 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d74b21d2d7105b5e1e755662f34cb2a2 in 1247ms, sequenceid=312, compaction requested=false 2024-11-25T03:16:14,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:14,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:14,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-25T03:16:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-25T03:16:14,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-25T03:16:14,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8600 sec 2024-11-25T03:16:14,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8640 sec 2024-11-25T03:16:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:14,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-25T03:16:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:14,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/98e11cbe36cb4815a0b8cdb17aa94038 is 50, key is test_row_0/A:col10/1732504574360/Put/seqid=0 2024-11-25T03:16:14,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742294_1470 (size=14741) 2024-11-25T03:16:14,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504634387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504634390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504634510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504634510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:16:14,630 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:14,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-25T03:16:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-25T03:16:14,632 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:14,633 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:14,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504634730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504634730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-25T03:16:14,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/98e11cbe36cb4815a0b8cdb17aa94038 2024-11-25T03:16:14,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/aa15c32ea4f642928eaa5b0951e2ce2b is 50, key is test_row_0/B:col10/1732504574360/Put/seqid=0 2024-11-25T03:16:14,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-25T03:16:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:14,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742295_1471 (size=12301) 2024-11-25T03:16:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:14,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:14,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-25T03:16:14,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:14,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:14,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:14,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:14,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-25T03:16:15,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504635040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:15,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504635040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:15,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-25T03:16:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:15,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:15,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/aa15c32ea4f642928eaa5b0951e2ce2b 2024-11-25T03:16:15,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/33af8fff881e4300a17003c823010909 is 50, key is test_row_0/C:col10/1732504574360/Put/seqid=0 2024-11-25T03:16:15,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742296_1472 (size=12301) 2024-11-25T03:16:15,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/33af8fff881e4300a17003c823010909 2024-11-25T03:16:15,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/98e11cbe36cb4815a0b8cdb17aa94038 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/98e11cbe36cb4815a0b8cdb17aa94038 2024-11-25T03:16:15,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/98e11cbe36cb4815a0b8cdb17aa94038, entries=200, sequenceid=351, filesize=14.4 K 2024-11-25T03:16:15,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/aa15c32ea4f642928eaa5b0951e2ce2b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/aa15c32ea4f642928eaa5b0951e2ce2b 2024-11-25T03:16:15,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/aa15c32ea4f642928eaa5b0951e2ce2b, entries=150, sequenceid=351, filesize=12.0 K 2024-11-25T03:16:15,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/33af8fff881e4300a17003c823010909 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/33af8fff881e4300a17003c823010909 2024-11-25T03:16:15,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/33af8fff881e4300a17003c823010909, entries=150, sequenceid=351, filesize=12.0 K 2024-11-25T03:16:15,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for d74b21d2d7105b5e1e755662f34cb2a2 in 886ms, sequenceid=351, compaction requested=true 2024-11-25T03:16:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:15,247 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:15,247 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:15,248 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:15,248 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:15,248 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:15,249 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/888d6c67fa00434b9266dab8d51c7dcb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/a371aaac6d0741f591dc294039defbc1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/98e11cbe36cb4815a0b8cdb17aa94038] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=38.9 K 2024-11-25T03:16:15,249 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:15,249 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:15,249 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:15,249 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/6feb4802efc54047b7fc67957bce22b4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/294ef3d0bad5410ab8d2816cd5d63c6e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/aa15c32ea4f642928eaa5b0951e2ce2b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.5 K 2024-11-25T03:16:15,250 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888d6c67fa00434b9266dab8d51c7dcb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732504571764 2024-11-25T03:16:15,250 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 6feb4802efc54047b7fc67957bce22b4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732504571764 2024-11-25T03:16:15,250 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting a371aaac6d0741f591dc294039defbc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732504571834 2024-11-25T03:16:15,250 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 294ef3d0bad5410ab8d2816cd5d63c6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732504571834 2024-11-25T03:16:15,251 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting aa15c32ea4f642928eaa5b0951e2ce2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732504574235 2024-11-25T03:16:15,251 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e11cbe36cb4815a0b8cdb17aa94038, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732504574222 2024-11-25T03:16:15,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-25T03:16:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:15,259 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-25T03:16:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-25T03:16:15,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/c1738d4169b34817bc9a1b1e3d71bfb7 is 50, key is test_row_0/A:col10/1732504574374/Put/seqid=0 2024-11-25T03:16:15,281 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:15,281 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:15,281 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/cf2857c667a043bf8492730aa39c2917 is 50, key is test_row_0/B:col10/1732504574360/Put/seqid=0 2024-11-25T03:16:15,281 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/6301ef55721643529d42cc61a3ea8288 is 50, key is test_row_0/A:col10/1732504574360/Put/seqid=0 2024-11-25T03:16:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742297_1473 (size=12301) 2024-11-25T03:16:15,296 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/c1738d4169b34817bc9a1b1e3d71bfb7 2024-11-25T03:16:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/d3d90e23dfbe41a6a90fdcb775e70ca3 is 50, key is test_row_0/B:col10/1732504574374/Put/seqid=0 2024-11-25T03:16:15,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742298_1474 (size=12881) 2024-11-25T03:16:15,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742299_1475 (size=12881) 2024-11-25T03:16:15,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742300_1476 (size=12301) 2024-11-25T03:16:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:15,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504575658, max timestamp=1732504575693Num uniques 859; 2024-11-25T03:16:15,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:16:15,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504575658, max timestamp=1732504575693Num uniques 859; 2024-11-25T03:16:15,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:16:15,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732504575658, max timestamp=1732504575693Num uniques 859; 2024-11-25T03:16:15,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T03:16:15,723 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/cf2857c667a043bf8492730aa39c2917 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/cf2857c667a043bf8492730aa39c2917 2024-11-25T03:16:15,726 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into cf2857c667a043bf8492730aa39c2917(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:15,727 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:15,727 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=13, startTime=1732504575247; duration=0sec 2024-11-25T03:16:15,727 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:15,727 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:15,727 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:15,731 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:15,731 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:15,732 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:15,732 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/adcc8e0d3105415696eeb12294bdad04, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/eccd246a346641c5bdeff3a31c78983e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/33af8fff881e4300a17003c823010909] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.5 K 2024-11-25T03:16:15,733 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting adcc8e0d3105415696eeb12294bdad04, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732504571764 2024-11-25T03:16:15,733 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/d3d90e23dfbe41a6a90fdcb775e70ca3 2024-11-25T03:16:15,733 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting eccd246a346641c5bdeff3a31c78983e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732504571834 2024-11-25T03:16:15,734 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 33af8fff881e4300a17003c823010909, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732504574235 2024-11-25T03:16:15,738 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/6301ef55721643529d42cc61a3ea8288 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6301ef55721643529d42cc61a3ea8288 2024-11-25T03:16:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/622a25f50cb444c68df4cd2e249d35b8 is 50, key is test_row_0/C:col10/1732504574374/Put/seqid=0 2024-11-25T03:16:15,745 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into 6301ef55721643529d42cc61a3ea8288(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:15,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:15,745 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=13, startTime=1732504575247; duration=0sec 2024-11-25T03:16:15,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:15,745 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:15,745 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:15,746 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/c418925d5c3a4bc082305c80409e723b is 50, key is test_row_0/C:col10/1732504574360/Put/seqid=0 2024-11-25T03:16:15,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742301_1477 (size=12301) 2024-11-25T03:16:15,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742302_1478 (size=12881) 2024-11-25T03:16:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-25T03:16:15,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504635803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:15,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504635807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:15,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504635910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504635930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504636130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,158 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/622a25f50cb444c68df4cd2e249d35b8 2024-11-25T03:16:16,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504636150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/c1738d4169b34817bc9a1b1e3d71bfb7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c1738d4169b34817bc9a1b1e3d71bfb7 2024-11-25T03:16:16,168 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/c418925d5c3a4bc082305c80409e723b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c418925d5c3a4bc082305c80409e723b 2024-11-25T03:16:16,169 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c1738d4169b34817bc9a1b1e3d71bfb7, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T03:16:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/d3d90e23dfbe41a6a90fdcb775e70ca3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d3d90e23dfbe41a6a90fdcb775e70ca3 2024-11-25T03:16:16,173 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into c418925d5c3a4bc082305c80409e723b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:16,173 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:16,173 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=13, startTime=1732504575247; duration=0sec 2024-11-25T03:16:16,173 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:16,173 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:16,177 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d3d90e23dfbe41a6a90fdcb775e70ca3, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T03:16:16,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/622a25f50cb444c68df4cd2e249d35b8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/622a25f50cb444c68df4cd2e249d35b8 2024-11-25T03:16:16,182 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/622a25f50cb444c68df4cd2e249d35b8, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T03:16:16,183 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for d74b21d2d7105b5e1e755662f34cb2a2 in 924ms, sequenceid=360, compaction requested=false 2024-11-25T03:16:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-25T03:16:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-25T03:16:16,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-25T03:16:16,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5520 sec 2024-11-25T03:16:16,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.5580 sec 2024-11-25T03:16:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:16,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-25T03:16:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T03:16:16,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/7d8cf17597e149c7b4b028cd8a7d0976 is 50, key is test_row_0/A:col10/1732504576445/Put/seqid=0 2024-11-25T03:16:16,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742303_1479 (size=14741) 2024-11-25T03:16:16,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504636518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504636520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504636641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504636641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-25T03:16:16,780 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:16,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-25T03:16:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-25T03:16:16,783 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:16,784 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:16,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:16,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/7d8cf17597e149c7b4b028cd8a7d0976 2024-11-25T03:16:16,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/691775f6377a4e998a4429612a75bd37 is 50, key is test_row_0/B:col10/1732504576445/Put/seqid=0 2024-11-25T03:16:16,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504636860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504636860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:16,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742304_1480 (size=12301) 2024-11-25T03:16:16,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/691775f6377a4e998a4429612a75bd37 2024-11-25T03:16:16,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/7165293846af442ea938c3ff8dc13878 is 50, key is test_row_0/C:col10/1732504576445/Put/seqid=0 2024-11-25T03:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-25T03:16:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742305_1481 (size=12301) 2024-11-25T03:16:16,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-25T03:16:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:16,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:17,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-25T03:16:17,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-25T03:16:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504637170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504637180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:17,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-25T03:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:17,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/7165293846af442ea938c3ff8dc13878 2024-11-25T03:16:17,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/7d8cf17597e149c7b4b028cd8a7d0976 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7d8cf17597e149c7b4b028cd8a7d0976 2024-11-25T03:16:17,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7d8cf17597e149c7b4b028cd8a7d0976, entries=200, sequenceid=401, filesize=14.4 K 2024-11-25T03:16:17,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/691775f6377a4e998a4429612a75bd37 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/691775f6377a4e998a4429612a75bd37 2024-11-25T03:16:17,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/691775f6377a4e998a4429612a75bd37, entries=150, sequenceid=401, filesize=12.0 K 2024-11-25T03:16:17,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/7165293846af442ea938c3ff8dc13878 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/7165293846af442ea938c3ff8dc13878 2024-11-25T03:16:17,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/7165293846af442ea938c3ff8dc13878, entries=150, sequenceid=401, filesize=12.0 K 2024-11-25T03:16:17,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=40.25 KB/41220 for d74b21d2d7105b5e1e755662f34cb2a2 in 876ms, sequenceid=401, compaction requested=true 2024-11-25T03:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:17,323 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:17,323 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:17,324 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:17,324 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:17,324 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:17,324 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:17,324 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,324 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,324 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6301ef55721643529d42cc61a3ea8288, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c1738d4169b34817bc9a1b1e3d71bfb7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7d8cf17597e149c7b4b028cd8a7d0976] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=39.0 K 2024-11-25T03:16:17,324 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/cf2857c667a043bf8492730aa39c2917, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d3d90e23dfbe41a6a90fdcb775e70ca3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/691775f6377a4e998a4429612a75bd37] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.6 K 2024-11-25T03:16:17,324 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6301ef55721643529d42cc61a3ea8288, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732504574235 2024-11-25T03:16:17,324 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2857c667a043bf8492730aa39c2917, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732504574235 2024-11-25T03:16:17,325 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1738d4169b34817bc9a1b1e3d71bfb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504574374 2024-11-25T03:16:17,325 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d90e23dfbe41a6a90fdcb775e70ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504574374 2024-11-25T03:16:17,325 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8cf17597e149c7b4b028cd8a7d0976, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732504575783 2024-11-25T03:16:17,325 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 691775f6377a4e998a4429612a75bd37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732504575783 2024-11-25T03:16:17,346 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:17,346 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/4e37d23844b14535bbdb3c6b70240d5d is 50, key is test_row_0/B:col10/1732504576445/Put/seqid=0 2024-11-25T03:16:17,353 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:17,353 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/d0bb0e531e294f21ba04a9ea2031e49a is 50, key is test_row_0/A:col10/1732504576445/Put/seqid=0 2024-11-25T03:16:17,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742306_1482 (size=12983) 2024-11-25T03:16:17,367 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/4e37d23844b14535bbdb3c6b70240d5d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4e37d23844b14535bbdb3c6b70240d5d 2024-11-25T03:16:17,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742307_1483 (size=12983) 2024-11-25T03:16:17,372 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/d0bb0e531e294f21ba04a9ea2031e49a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/d0bb0e531e294f21ba04a9ea2031e49a 2024-11-25T03:16:17,372 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into 4e37d23844b14535bbdb3c6b70240d5d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:17,372 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:17,373 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=13, startTime=1732504577323; duration=0sec 2024-11-25T03:16:17,373 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:17,373 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:17,373 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:17,375 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:17,375 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:17,375 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,375 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c418925d5c3a4bc082305c80409e723b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/622a25f50cb444c68df4cd2e249d35b8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/7165293846af442ea938c3ff8dc13878] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.6 K 2024-11-25T03:16:17,375 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting c418925d5c3a4bc082305c80409e723b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732504574235 2024-11-25T03:16:17,376 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 622a25f50cb444c68df4cd2e249d35b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732504574374 2024-11-25T03:16:17,377 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into d0bb0e531e294f21ba04a9ea2031e49a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:17,377 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:17,377 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=13, startTime=1732504577322; duration=0sec 2024-11-25T03:16:17,378 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:17,378 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:17,384 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 7165293846af442ea938c3ff8dc13878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732504575783 2024-11-25T03:16:17,398 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:17,399 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/88328216d22444c692570b7b28653c2a is 50, key is test_row_0/C:col10/1732504576445/Put/seqid=0 2024-11-25T03:16:17,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-25T03:16:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,400 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-25T03:16:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-25T03:16:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/8b752431e849406c8c73cc755bb3e166 is 50, key is test_row_0/A:col10/1732504576517/Put/seqid=0 2024-11-25T03:16:17,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742308_1484 (size=12983) 2024-11-25T03:16:17,430 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/88328216d22444c692570b7b28653c2a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/88328216d22444c692570b7b28653c2a 2024-11-25T03:16:17,436 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into 88328216d22444c692570b7b28653c2a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:17,436 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:17,436 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=13, startTime=1732504577323; duration=0sec 2024-11-25T03:16:17,436 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:17,436 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:17,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742309_1485 (size=12301) 2024-11-25T03:16:17,439 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/8b752431e849406c8c73cc755bb3e166 2024-11-25T03:16:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5698763ee539446f977cb0907492576a is 50, key is test_row_0/B:col10/1732504576517/Put/seqid=0 2024-11-25T03:16:17,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742310_1486 (size=12301) 2024-11-25T03:16:17,470 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5698763ee539446f977cb0907492576a 2024-11-25T03:16:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/b0c272acdaae402e85961bc9b151afed is 50, key is test_row_0/C:col10/1732504576517/Put/seqid=0 2024-11-25T03:16:17,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742311_1487 (size=12301) 2024-11-25T03:16:17,495 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/b0c272acdaae402e85961bc9b151afed 2024-11-25T03:16:17,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/8b752431e849406c8c73cc755bb3e166 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/8b752431e849406c8c73cc755bb3e166 2024-11-25T03:16:17,505 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/8b752431e849406c8c73cc755bb3e166, entries=150, sequenceid=412, filesize=12.0 K 2024-11-25T03:16:17,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5698763ee539446f977cb0907492576a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5698763ee539446f977cb0907492576a 2024-11-25T03:16:17,511 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5698763ee539446f977cb0907492576a, entries=150, sequenceid=412, filesize=12.0 K 2024-11-25T03:16:17,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/b0c272acdaae402e85961bc9b151afed as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/b0c272acdaae402e85961bc9b151afed 2024-11-25T03:16:17,521 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/b0c272acdaae402e85961bc9b151afed, entries=150, sequenceid=412, filesize=12.0 K 2024-11-25T03:16:17,522 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for d74b21d2d7105b5e1e755662f34cb2a2 in 123ms, sequenceid=412, compaction requested=false 2024-11-25T03:16:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-25T03:16:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-25T03:16:17,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-25T03:16:17,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 741 msec 2024-11-25T03:16:17,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 746 msec 2024-11-25T03:16:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:17,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:17,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/9a58677464c34a1085340e507120d5bd is 50, key is test_row_0/A:col10/1732504577709/Put/seqid=0 2024-11-25T03:16:17,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742312_1488 (size=17181) 2024-11-25T03:16:17,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/9a58677464c34a1085340e507120d5bd 2024-11-25T03:16:17,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/a0ad69727b184ef69b0d8d720f5b5219 is 50, key is test_row_0/B:col10/1732504577709/Put/seqid=0 2024-11-25T03:16:17,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742313_1489 (size=12301) 2024-11-25T03:16:17,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504577758, max timestamp=1732504577829Num uniques 859; 2024-11-25T03:16:17,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:17,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732504577758, max timestamp=1732504577829Num uniques 859; 2024-11-25T03:16:17,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T03:16:17,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504577758, max timestamp=1732504577829Num uniques 859; 2024-11-25T03:16:17,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-25T03:16:17,921 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:17,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-25T03:16:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-25T03:16:17,925 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:17,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:17,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:17,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504637934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:17,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504637947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-25T03:16:18,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504638050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504638070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/a0ad69727b184ef69b0d8d720f5b5219 2024-11-25T03:16:18,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/5cbc746d20f04ef89ba03a5c6af38765 is 50, key is test_row_0/C:col10/1732504577709/Put/seqid=0 2024-11-25T03:16:18,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742314_1490 (size=12301) 2024-11-25T03:16:18,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-25T03:16:18,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504638270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504638290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:18,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:18,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-25T03:16:18,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/5cbc746d20f04ef89ba03a5c6af38765 2024-11-25T03:16:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504638590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/9a58677464c34a1085340e507120d5bd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a58677464c34a1085340e507120d5bd 2024-11-25T03:16:18,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a58677464c34a1085340e507120d5bd, entries=250, sequenceid=424, filesize=16.8 K 2024-11-25T03:16:18,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/a0ad69727b184ef69b0d8d720f5b5219 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a0ad69727b184ef69b0d8d720f5b5219 2024-11-25T03:16:18,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a0ad69727b184ef69b0d8d720f5b5219, entries=150, sequenceid=424, filesize=12.0 K 2024-11-25T03:16:18,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/5cbc746d20f04ef89ba03a5c6af38765 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5cbc746d20f04ef89ba03a5c6af38765 2024-11-25T03:16:18,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5cbc746d20f04ef89ba03a5c6af38765, entries=150, sequenceid=424, filesize=12.0 K 2024-11-25T03:16:18,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for d74b21d2d7105b5e1e755662f34cb2a2 in 903ms, sequenceid=424, compaction requested=true 2024-11-25T03:16:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:18,616 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:18,616 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:18,617 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:18,617 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:18,617 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:18,617 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:18,617 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,617 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,617 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/d0bb0e531e294f21ba04a9ea2031e49a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/8b752431e849406c8c73cc755bb3e166, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a58677464c34a1085340e507120d5bd] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=41.5 K 2024-11-25T03:16:18,617 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4e37d23844b14535bbdb3c6b70240d5d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5698763ee539446f977cb0907492576a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a0ad69727b184ef69b0d8d720f5b5219] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.7 K 2024-11-25T03:16:18,618 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e37d23844b14535bbdb3c6b70240d5d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732504575783 2024-11-25T03:16:18,618 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0bb0e531e294f21ba04a9ea2031e49a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732504575783 2024-11-25T03:16:18,618 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5698763ee539446f977cb0907492576a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732504576483 2024-11-25T03:16:18,618 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b752431e849406c8c73cc755bb3e166, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732504576483 2024-11-25T03:16:18,618 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ad69727b184ef69b0d8d720f5b5219, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732504577699 2024-11-25T03:16:18,618 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a58677464c34a1085340e507120d5bd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732504577690 2024-11-25T03:16:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:18,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-25T03:16:18,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:18,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:18,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:18,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:18,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:18,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:18,625 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:18,625 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5fdc7b96d24d4a19980410c787e2428f is 50, key is test_row_0/B:col10/1732504577709/Put/seqid=0 2024-11-25T03:16:18,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/f80c04bb499c459e9b2dbbed97cf012c is 50, key is test_row_0/A:col10/1732504577923/Put/seqid=0 2024-11-25T03:16:18,630 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:18,631 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/aca2fe78c36c432db337b046f6fb9699 is 50, key is test_row_0/A:col10/1732504577709/Put/seqid=0 2024-11-25T03:16:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742315_1491 (size=13085) 2024-11-25T03:16:18,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742316_1492 (size=14741) 2024-11-25T03:16:18,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742317_1493 (size=13085) 2024-11-25T03:16:18,644 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/aca2fe78c36c432db337b046f6fb9699 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/aca2fe78c36c432db337b046f6fb9699 2024-11-25T03:16:18,648 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into aca2fe78c36c432db337b046f6fb9699(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:18,648 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:18,648 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=13, startTime=1732504578616; duration=0sec 2024-11-25T03:16:18,649 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:18,649 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:18,649 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:18,649 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:18,650 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:18,650 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,650 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/88328216d22444c692570b7b28653c2a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/b0c272acdaae402e85961bc9b151afed, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5cbc746d20f04ef89ba03a5c6af38765] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.7 K 2024-11-25T03:16:18,650 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88328216d22444c692570b7b28653c2a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732504575783 2024-11-25T03:16:18,650 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c272acdaae402e85961bc9b151afed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732504576483 2024-11-25T03:16:18,650 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbc746d20f04ef89ba03a5c6af38765, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732504577699 2024-11-25T03:16:18,656 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:18,657 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/5a65994ab300449f9215725cb40627ff is 50, key is test_row_0/C:col10/1732504577709/Put/seqid=0 2024-11-25T03:16:18,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742318_1494 (size=13085) 2024-11-25T03:16:18,664 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/5a65994ab300449f9215725cb40627ff as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5a65994ab300449f9215725cb40627ff 2024-11-25T03:16:18,667 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into 5a65994ab300449f9215725cb40627ff(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:18,668 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:18,668 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=13, startTime=1732504578616; duration=0sec 2024-11-25T03:16:18,668 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:18,668 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:18,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:18,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:18,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504638711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:18,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504638830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:18,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:18,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:18,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:18,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:18,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:18,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,037 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/5fdc7b96d24d4a19980410c787e2428f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5fdc7b96d24d4a19980410c787e2428f 2024-11-25T03:16:19,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/f80c04bb499c459e9b2dbbed97cf012c 2024-11-25T03:16:19,042 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into 5fdc7b96d24d4a19980410c787e2428f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:19,042 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:19,042 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=13, startTime=1732504578616; duration=0sec 2024-11-25T03:16:19,042 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:19,042 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:19,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/475455259b13472aa374675b30875308 is 50, key is test_row_0/B:col10/1732504577923/Put/seqid=0 2024-11-25T03:16:19,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742319_1495 (size=12301) 2024-11-25T03:16:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504639040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-25T03:16:19,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504639110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:19,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:19,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:19,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504639359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:19,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/475455259b13472aa374675b30875308 2024-11-25T03:16:19,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/3680a189140348599362ad6545872681 is 50, key is test_row_0/C:col10/1732504577923/Put/seqid=0 2024-11-25T03:16:19,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:19,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:19,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742320_1496 (size=12301) 2024-11-25T03:16:19,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/3680a189140348599362ad6545872681 2024-11-25T03:16:19,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/f80c04bb499c459e9b2dbbed97cf012c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f80c04bb499c459e9b2dbbed97cf012c 2024-11-25T03:16:19,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f80c04bb499c459e9b2dbbed97cf012c, entries=200, sequenceid=461, filesize=14.4 K 2024-11-25T03:16:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/475455259b13472aa374675b30875308 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/475455259b13472aa374675b30875308 2024-11-25T03:16:19,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/475455259b13472aa374675b30875308, entries=150, sequenceid=461, filesize=12.0 K 2024-11-25T03:16:19,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/3680a189140348599362ad6545872681 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/3680a189140348599362ad6545872681 2024-11-25T03:16:19,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/3680a189140348599362ad6545872681, entries=150, sequenceid=461, filesize=12.0 K 2024-11-25T03:16:19,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for d74b21d2d7105b5e1e755662f34cb2a2 in 1255ms, sequenceid=461, compaction requested=false 2024-11-25T03:16:19,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:19,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:16:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:19,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/ebf04611ffe449d283079078a854cf55 is 50, key is test_row_0/A:col10/1732504578710/Put/seqid=0 2024-11-25T03:16:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742321_1497 (size=12301) 2024-11-25T03:16:19,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:19,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504580004, max timestamp=1732504580051Num uniques 859; 2024-11-25T03:16:20,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504580004, max timestamp=1732504580051Num uniques 859; 2024-11-25T03:16:20,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:20,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:20,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504580004, max timestamp=1732504580051Num uniques 859; 2024-11-25T03:16:20,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:20,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-25T03:16:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504640170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504640174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504640280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/ebf04611ffe449d283079078a854cf55 2024-11-25T03:16:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504640289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/22b769c6653e4c40bb170e54e5ae673b is 50, key is test_row_0/B:col10/1732504578710/Put/seqid=0 2024-11-25T03:16:20,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742322_1498 (size=12301) 2024-11-25T03:16:20,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504640500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504640510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/22b769c6653e4c40bb170e54e5ae673b 2024-11-25T03:16:20,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/84dc50c52d20468187d474157cdc2462 is 50, key is test_row_0/C:col10/1732504578710/Put/seqid=0 2024-11-25T03:16:20,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742323_1499 (size=12301) 2024-11-25T03:16:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36376 deadline: 1732504640820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36326 deadline: 1732504640830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 , the old value is region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., hostname=38a5d2077268,43357,1732504425108, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:16:20,862 INFO [Thread-1926 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1926. 2024-11-25T03:16:20,862 INFO [Thread-1922 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1922. 2024-11-25T03:16:20,862 DEBUG [Thread-1922 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:20,862 DEBUG [Thread-1926 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:20,862 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,862 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,862 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,862 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:20,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:20,869 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-11-25T03:16:20,869 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:20,869 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,869 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:20,874 INFO [Thread-1924 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1924. 2024-11-25T03:16:20,874 DEBUG [Thread-1924 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:20,874 DEBUG [Thread-1924 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,874 DEBUG [Thread-1924 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:20,875 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-25T03:16:20,875 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:20,875 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,875 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:20,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:20,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:20,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:21,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/84dc50c52d20468187d474157cdc2462 2024-11-25T03:16:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/ebf04611ffe449d283079078a854cf55 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/ebf04611ffe449d283079078a854cf55 2024-11-25T03:16:21,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/ebf04611ffe449d283079078a854cf55, entries=150, sequenceid=475, filesize=12.0 K 2024-11-25T03:16:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/22b769c6653e4c40bb170e54e5ae673b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/22b769c6653e4c40bb170e54e5ae673b 2024-11-25T03:16:21,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/22b769c6653e4c40bb170e54e5ae673b, entries=150, sequenceid=475, filesize=12.0 K 2024-11-25T03:16:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/84dc50c52d20468187d474157cdc2462 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/84dc50c52d20468187d474157cdc2462 2024-11-25T03:16:21,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/84dc50c52d20468187d474157cdc2462, entries=150, sequenceid=475, filesize=12.0 K 2024-11-25T03:16:21,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d74b21d2d7105b5e1e755662f34cb2a2 in 1256ms, sequenceid=475, compaction requested=true 2024-11-25T03:16:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d74b21d2d7105b5e1e755662f34cb2a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:21,138 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:21,138 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:21,139 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:21,139 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:21,139 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/A is initiating minor compaction (all files) 2024-11-25T03:16:21,139 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/B is initiating minor compaction (all files) 2024-11-25T03:16:21,139 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/A in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:21,139 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/B in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:21,139 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/aca2fe78c36c432db337b046f6fb9699, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f80c04bb499c459e9b2dbbed97cf012c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/ebf04611ffe449d283079078a854cf55] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=39.2 K 2024-11-25T03:16:21,139 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5fdc7b96d24d4a19980410c787e2428f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/475455259b13472aa374675b30875308, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/22b769c6653e4c40bb170e54e5ae673b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.8 K 2024-11-25T03:16:21,139 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting aca2fe78c36c432db337b046f6fb9699, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732504577699 2024-11-25T03:16:21,139 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fdc7b96d24d4a19980410c787e2428f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732504577699 2024-11-25T03:16:21,140 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80c04bb499c459e9b2dbbed97cf012c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732504577917 2024-11-25T03:16:21,140 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 475455259b13472aa374675b30875308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732504577917 2024-11-25T03:16:21,140 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf04611ffe449d283079078a854cf55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732504578701 2024-11-25T03:16:21,140 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b769c6653e4c40bb170e54e5ae673b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732504578701 2024-11-25T03:16:21,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-25T03:16:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:21,141 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-25T03:16:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/280c9961d05245f7b0961138cdf22a26 is 50, key is test_row_0/A:col10/1732504580169/Put/seqid=0 2024-11-25T03:16:21,147 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:21,147 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ab6ec3aa2d8646099561c6282fd2032e is 50, key is test_row_0/B:col10/1732504578710/Put/seqid=0 2024-11-25T03:16:21,152 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#A#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:21,152 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/0de62ce8aeb1475191871e856eb9246e is 50, key is test_row_0/A:col10/1732504578710/Put/seqid=0 2024-11-25T03:16:21,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742324_1500 (size=12301) 2024-11-25T03:16:21,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742325_1501 (size=13187) 2024-11-25T03:16:21,157 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/280c9961d05245f7b0961138cdf22a26 2024-11-25T03:16:21,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742326_1502 (size=13187) 2024-11-25T03:16:21,162 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/0de62ce8aeb1475191871e856eb9246e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/0de62ce8aeb1475191871e856eb9246e 2024-11-25T03:16:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/497d15c7b14d496dbf93178edf3f3897 is 50, key is test_row_0/B:col10/1732504580169/Put/seqid=0 2024-11-25T03:16:21,165 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/A of d74b21d2d7105b5e1e755662f34cb2a2 into 0de62ce8aeb1475191871e856eb9246e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:21,165 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:21,165 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/A, priority=13, startTime=1732504581138; duration=0sec 2024-11-25T03:16:21,166 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:21,166 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:A 2024-11-25T03:16:21,166 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:21,167 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:21,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742327_1503 (size=12301) 2024-11-25T03:16:21,167 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): d74b21d2d7105b5e1e755662f34cb2a2/C is initiating minor compaction (all files) 2024-11-25T03:16:21,167 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d74b21d2d7105b5e1e755662f34cb2a2/C in TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:21,167 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5a65994ab300449f9215725cb40627ff, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/3680a189140348599362ad6545872681, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/84dc50c52d20468187d474157cdc2462] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp, totalSize=36.8 K 2024-11-25T03:16:21,167 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a65994ab300449f9215725cb40627ff, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732504577699 2024-11-25T03:16:21,167 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3680a189140348599362ad6545872681, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732504577917 2024-11-25T03:16:21,168 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84dc50c52d20468187d474157cdc2462, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732504578701 2024-11-25T03:16:21,173 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d74b21d2d7105b5e1e755662f34cb2a2#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:21,173 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/e6138479e6864702b3b9aad483577bfb is 50, key is test_row_0/C:col10/1732504578710/Put/seqid=0 2024-11-25T03:16:21,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742328_1504 (size=13187) 2024-11-25T03:16:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:21,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. as already flushing 2024-11-25T03:16:21,342 INFO [Thread-1913 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1913. 2024-11-25T03:16:21,342 DEBUG [Thread-1913 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:21,342 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:21,342 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:21,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:21,351 INFO [Thread-1915 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1915. 2024-11-25T03:16:21,351 DEBUG [Thread-1915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:21,351 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:21,351 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:21,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:21,559 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ab6ec3aa2d8646099561c6282fd2032e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ab6ec3aa2d8646099561c6282fd2032e 2024-11-25T03:16:21,562 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/B of d74b21d2d7105b5e1e755662f34cb2a2 into ab6ec3aa2d8646099561c6282fd2032e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:21,562 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:21,562 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/B, priority=13, startTime=1732504581138; duration=0sec 2024-11-25T03:16:21,562 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:21,562 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:B 2024-11-25T03:16:21,567 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/497d15c7b14d496dbf93178edf3f3897 2024-11-25T03:16:21,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/0383604f0d6e4a399ab4fa09570fb4cd is 50, key is test_row_0/C:col10/1732504580169/Put/seqid=0 2024-11-25T03:16:21,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742329_1505 (size=12301) 2024-11-25T03:16:21,581 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/e6138479e6864702b3b9aad483577bfb as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/e6138479e6864702b3b9aad483577bfb 2024-11-25T03:16:21,584 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d74b21d2d7105b5e1e755662f34cb2a2/C of d74b21d2d7105b5e1e755662f34cb2a2 into e6138479e6864702b3b9aad483577bfb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:21,585 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:21,585 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., storeName=d74b21d2d7105b5e1e755662f34cb2a2/C, priority=13, startTime=1732504581138; duration=0sec 2024-11-25T03:16:21,585 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:21,585 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d74b21d2d7105b5e1e755662f34cb2a2:C 2024-11-25T03:16:21,976 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/0383604f0d6e4a399ab4fa09570fb4cd 2024-11-25T03:16:21,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/280c9961d05245f7b0961138cdf22a26 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/280c9961d05245f7b0961138cdf22a26 2024-11-25T03:16:21,983 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/280c9961d05245f7b0961138cdf22a26, entries=150, sequenceid=511, filesize=12.0 K 2024-11-25T03:16:21,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/497d15c7b14d496dbf93178edf3f3897 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/497d15c7b14d496dbf93178edf3f3897 2024-11-25T03:16:21,986 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/497d15c7b14d496dbf93178edf3f3897, entries=150, sequenceid=511, filesize=12.0 K 2024-11-25T03:16:21,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/0383604f0d6e4a399ab4fa09570fb4cd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/0383604f0d6e4a399ab4fa09570fb4cd 2024-11-25T03:16:21,990 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/0383604f0d6e4a399ab4fa09570fb4cd, entries=150, sequenceid=511, filesize=12.0 K 2024-11-25T03:16:21,991 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=13.42 KB/13740 for d74b21d2d7105b5e1e755662f34cb2a2 in 850ms, sequenceid=511, compaction requested=false 2024-11-25T03:16:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for d74b21d2d7105b5e1e755662f34cb2a2: 2024-11-25T03:16:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-25T03:16:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-25T03:16:21,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-25T03:16:21,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0660 sec 2024-11-25T03:16:21,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 4.0730 sec 2024-11-25T03:16:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-25T03:16:22,090 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:24,301 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-11-25T03:16:24,301 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:24,301 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,302 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:24,322 INFO [Thread-1911 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1911. 2024-11-25T03:16:24,322 DEBUG [Thread-1911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:24,322 DEBUG [Thread-1911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,322 DEBUG [Thread-1911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:24,331 INFO [Thread-1917 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1917. 2024-11-25T03:16:24,331 DEBUG [Thread-1917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:24,331 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,331 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 214 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1165 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3488 rows 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1162 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3486 rows 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1161 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3480 rows 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1161 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3480 rows 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1160 2024-11-25T03:16:24,331 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3477 rows 2024-11-25T03:16:24,332 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:16:24,332 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:16:24,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:24,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:24,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:16:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-25T03:16:24,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504584335"}]},"ts":"1732504584335"} 2024-11-25T03:16:24,336 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:16:24,336 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:16:24,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:16:24,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, UNASSIGN}] 2024-11-25T03:16:24,339 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, UNASSIGN 2024-11-25T03:16:24,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d74b21d2d7105b5e1e755662f34cb2a2, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:16:24,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, UNASSIGN because future has completed 2024-11-25T03:16:24,341 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:16:24,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:16:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-25T03:16:24,493 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing d74b21d2d7105b5e1e755662f34cb2a2, disabling compactions & flushes 2024-11-25T03:16:24,493 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. after waiting 0 ms 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:24,493 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing d74b21d2d7105b5e1e755662f34cb2a2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=A 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=B 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d74b21d2d7105b5e1e755662f34cb2a2, store=C 2024-11-25T03:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:24,496 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/206ba4d0a96f4e038a3082d23ad5126f is 50, key is test_row_0/A:col10/1732504584300/Put/seqid=0 2024-11-25T03:16:24,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742330_1506 (size=12301) 2024-11-25T03:16:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-25T03:16:24,900 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/206ba4d0a96f4e038a3082d23ad5126f 2024-11-25T03:16:24,906 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ef94fd1cf66343968e31fa7944694280 is 50, key is test_row_0/B:col10/1732504584300/Put/seqid=0 2024-11-25T03:16:24,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742331_1507 (size=12301) 2024-11-25T03:16:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-25T03:16:25,309 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ef94fd1cf66343968e31fa7944694280 2024-11-25T03:16:25,316 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/306dee7db24d4d1bace24c2bc9d4be58 is 50, key is test_row_0/C:col10/1732504584300/Put/seqid=0 2024-11-25T03:16:25,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742332_1508 (size=12301) 2024-11-25T03:16:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-25T03:16:25,719 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/306dee7db24d4d1bace24c2bc9d4be58 2024-11-25T03:16:25,723 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/A/206ba4d0a96f4e038a3082d23ad5126f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/206ba4d0a96f4e038a3082d23ad5126f 2024-11-25T03:16:25,726 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/206ba4d0a96f4e038a3082d23ad5126f, entries=150, sequenceid=522, filesize=12.0 K 2024-11-25T03:16:25,727 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/B/ef94fd1cf66343968e31fa7944694280 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ef94fd1cf66343968e31fa7944694280 2024-11-25T03:16:25,730 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ef94fd1cf66343968e31fa7944694280, entries=150, sequenceid=522, filesize=12.0 K 2024-11-25T03:16:25,730 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/.tmp/C/306dee7db24d4d1bace24c2bc9d4be58 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/306dee7db24d4d1bace24c2bc9d4be58 2024-11-25T03:16:25,733 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/306dee7db24d4d1bace24c2bc9d4be58, entries=150, sequenceid=522, filesize=12.0 K 2024-11-25T03:16:25,733 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d74b21d2d7105b5e1e755662f34cb2a2 in 1240ms, sequenceid=522, compaction requested=true 2024-11-25T03:16:25,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f528538b4a3740bb96df9faa145b1e2c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/28a0a7867b314002ae87755cc3b023af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/1aac8d4cd4224822a21dc719f9d299b5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/5e9bedbfe70c4f3899b743ee2dafd435, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/cb78ac46209644899253d3625cf1ddbb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/80b4bcb442314817899478b23272d90e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/15c76ede6ac0444e89d1944ced0fdc90, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c22260762ef443c291f43119c44c49ce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6312eb09b7e04f628ff8f73227eebf50, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a900485f66c4304bc12a3cd6e88d924, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7a316d47e55a4b96963c076521043651, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/14c456f6ef7c493a94a94c7d4c7c59cc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/3c2340a5b4e24d2ab8cd459f8f315b78, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/34c7fcb2deaf4991bdb431f643908686, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/888d6c67fa00434b9266dab8d51c7dcb, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/a371aaac6d0741f591dc294039defbc1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/98e11cbe36cb4815a0b8cdb17aa94038, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6301ef55721643529d42cc61a3ea8288, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c1738d4169b34817bc9a1b1e3d71bfb7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7d8cf17597e149c7b4b028cd8a7d0976, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/d0bb0e531e294f21ba04a9ea2031e49a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/8b752431e849406c8c73cc755bb3e166, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a58677464c34a1085340e507120d5bd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/aca2fe78c36c432db337b046f6fb9699, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f80c04bb499c459e9b2dbbed97cf012c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/ebf04611ffe449d283079078a854cf55] to archive 2024-11-25T03:16:25,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:16:25,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f528538b4a3740bb96df9faa145b1e2c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f528538b4a3740bb96df9faa145b1e2c 2024-11-25T03:16:25,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/28a0a7867b314002ae87755cc3b023af to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/28a0a7867b314002ae87755cc3b023af 2024-11-25T03:16:25,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/1aac8d4cd4224822a21dc719f9d299b5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/1aac8d4cd4224822a21dc719f9d299b5 2024-11-25T03:16:25,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/5e9bedbfe70c4f3899b743ee2dafd435 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/5e9bedbfe70c4f3899b743ee2dafd435 2024-11-25T03:16:25,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/cb78ac46209644899253d3625cf1ddbb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/cb78ac46209644899253d3625cf1ddbb 2024-11-25T03:16:25,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/80b4bcb442314817899478b23272d90e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/80b4bcb442314817899478b23272d90e 2024-11-25T03:16:25,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/15c76ede6ac0444e89d1944ced0fdc90 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/15c76ede6ac0444e89d1944ced0fdc90 2024-11-25T03:16:25,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c22260762ef443c291f43119c44c49ce to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c22260762ef443c291f43119c44c49ce 2024-11-25T03:16:25,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6312eb09b7e04f628ff8f73227eebf50 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6312eb09b7e04f628ff8f73227eebf50 2024-11-25T03:16:25,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a900485f66c4304bc12a3cd6e88d924 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a900485f66c4304bc12a3cd6e88d924 2024-11-25T03:16:25,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7a316d47e55a4b96963c076521043651 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7a316d47e55a4b96963c076521043651 2024-11-25T03:16:25,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/14c456f6ef7c493a94a94c7d4c7c59cc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/14c456f6ef7c493a94a94c7d4c7c59cc 2024-11-25T03:16:25,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/3c2340a5b4e24d2ab8cd459f8f315b78 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/3c2340a5b4e24d2ab8cd459f8f315b78 2024-11-25T03:16:25,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/34c7fcb2deaf4991bdb431f643908686 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/34c7fcb2deaf4991bdb431f643908686 2024-11-25T03:16:25,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/888d6c67fa00434b9266dab8d51c7dcb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/888d6c67fa00434b9266dab8d51c7dcb 2024-11-25T03:16:25,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/a371aaac6d0741f591dc294039defbc1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/a371aaac6d0741f591dc294039defbc1 2024-11-25T03:16:25,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/98e11cbe36cb4815a0b8cdb17aa94038 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/98e11cbe36cb4815a0b8cdb17aa94038 2024-11-25T03:16:25,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6301ef55721643529d42cc61a3ea8288 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/6301ef55721643529d42cc61a3ea8288 2024-11-25T03:16:25,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c1738d4169b34817bc9a1b1e3d71bfb7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/c1738d4169b34817bc9a1b1e3d71bfb7 2024-11-25T03:16:25,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7d8cf17597e149c7b4b028cd8a7d0976 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/7d8cf17597e149c7b4b028cd8a7d0976 2024-11-25T03:16:25,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/d0bb0e531e294f21ba04a9ea2031e49a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/d0bb0e531e294f21ba04a9ea2031e49a 2024-11-25T03:16:25,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/8b752431e849406c8c73cc755bb3e166 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/8b752431e849406c8c73cc755bb3e166 2024-11-25T03:16:25,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a58677464c34a1085340e507120d5bd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/9a58677464c34a1085340e507120d5bd 2024-11-25T03:16:25,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/aca2fe78c36c432db337b046f6fb9699 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/aca2fe78c36c432db337b046f6fb9699 2024-11-25T03:16:25,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f80c04bb499c459e9b2dbbed97cf012c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/f80c04bb499c459e9b2dbbed97cf012c 2024-11-25T03:16:25,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/ebf04611ffe449d283079078a854cf55 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/ebf04611ffe449d283079078a854cf55 2024-11-25T03:16:25,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/2c3fa3340f2a4629acaa4d6325b4bdfe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5ee149934c3d426a87351442bb2bf933, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/0a1961fcc6264adfbd31a8d44955488f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9d0965a505ea4cd283f97b76d0e3762c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/978ec1cdd0124922945c81a37660c19f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/923e79e503c54654bb0ae1c105d335ce, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4586565eb9a642cfb286cc15b4853f28, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9ddc49d4f5314c09afb524ebda8ac6b8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/af9ac7b8b0eb43e093c1161911457539, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ac3f503661474a74815875c9e362b8b7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ebc857bb25674cd295c9cf4678ae3bba, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a5021b3eddff4acf8eae5645c11263d5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/15250359885e4af8ac3c45cf3605f19d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/6feb4802efc54047b7fc67957bce22b4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d75892242cf04ac8a65e6de7aeefd189, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/294ef3d0bad5410ab8d2816cd5d63c6e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/cf2857c667a043bf8492730aa39c2917, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/aa15c32ea4f642928eaa5b0951e2ce2b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d3d90e23dfbe41a6a90fdcb775e70ca3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4e37d23844b14535bbdb3c6b70240d5d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/691775f6377a4e998a4429612a75bd37, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5698763ee539446f977cb0907492576a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5fdc7b96d24d4a19980410c787e2428f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a0ad69727b184ef69b0d8d720f5b5219, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/475455259b13472aa374675b30875308, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/22b769c6653e4c40bb170e54e5ae673b] to archive 2024-11-25T03:16:25,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:16:25,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/2c3fa3340f2a4629acaa4d6325b4bdfe to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/2c3fa3340f2a4629acaa4d6325b4bdfe 2024-11-25T03:16:25,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5ee149934c3d426a87351442bb2bf933 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5ee149934c3d426a87351442bb2bf933 2024-11-25T03:16:25,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/0a1961fcc6264adfbd31a8d44955488f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/0a1961fcc6264adfbd31a8d44955488f 2024-11-25T03:16:25,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9d0965a505ea4cd283f97b76d0e3762c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9d0965a505ea4cd283f97b76d0e3762c 2024-11-25T03:16:25,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/978ec1cdd0124922945c81a37660c19f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/978ec1cdd0124922945c81a37660c19f 2024-11-25T03:16:25,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/923e79e503c54654bb0ae1c105d335ce to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/923e79e503c54654bb0ae1c105d335ce 2024-11-25T03:16:25,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4586565eb9a642cfb286cc15b4853f28 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4586565eb9a642cfb286cc15b4853f28 2024-11-25T03:16:25,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9ddc49d4f5314c09afb524ebda8ac6b8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/9ddc49d4f5314c09afb524ebda8ac6b8 2024-11-25T03:16:25,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/af9ac7b8b0eb43e093c1161911457539 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/af9ac7b8b0eb43e093c1161911457539 2024-11-25T03:16:25,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ac3f503661474a74815875c9e362b8b7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ac3f503661474a74815875c9e362b8b7 2024-11-25T03:16:25,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ebc857bb25674cd295c9cf4678ae3bba to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ebc857bb25674cd295c9cf4678ae3bba 2024-11-25T03:16:25,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a5021b3eddff4acf8eae5645c11263d5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a5021b3eddff4acf8eae5645c11263d5 2024-11-25T03:16:25,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/15250359885e4af8ac3c45cf3605f19d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/15250359885e4af8ac3c45cf3605f19d 2024-11-25T03:16:25,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/6feb4802efc54047b7fc67957bce22b4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/6feb4802efc54047b7fc67957bce22b4 2024-11-25T03:16:25,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d75892242cf04ac8a65e6de7aeefd189 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d75892242cf04ac8a65e6de7aeefd189 2024-11-25T03:16:25,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/294ef3d0bad5410ab8d2816cd5d63c6e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/294ef3d0bad5410ab8d2816cd5d63c6e 2024-11-25T03:16:25,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/cf2857c667a043bf8492730aa39c2917 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/cf2857c667a043bf8492730aa39c2917 2024-11-25T03:16:25,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/aa15c32ea4f642928eaa5b0951e2ce2b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/aa15c32ea4f642928eaa5b0951e2ce2b 2024-11-25T03:16:25,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d3d90e23dfbe41a6a90fdcb775e70ca3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/d3d90e23dfbe41a6a90fdcb775e70ca3 2024-11-25T03:16:25,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4e37d23844b14535bbdb3c6b70240d5d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/4e37d23844b14535bbdb3c6b70240d5d 2024-11-25T03:16:25,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/691775f6377a4e998a4429612a75bd37 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/691775f6377a4e998a4429612a75bd37 2024-11-25T03:16:25,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5698763ee539446f977cb0907492576a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5698763ee539446f977cb0907492576a 2024-11-25T03:16:25,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5fdc7b96d24d4a19980410c787e2428f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/5fdc7b96d24d4a19980410c787e2428f 2024-11-25T03:16:25,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a0ad69727b184ef69b0d8d720f5b5219 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/a0ad69727b184ef69b0d8d720f5b5219 2024-11-25T03:16:25,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/475455259b13472aa374675b30875308 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/475455259b13472aa374675b30875308 2024-11-25T03:16:25,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/22b769c6653e4c40bb170e54e5ae673b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/22b769c6653e4c40bb170e54e5ae673b 2024-11-25T03:16:25,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/06f94fb0b5f24011b850ad64414ec877, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c199688239df426dae54e97c709b8c51, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/15c1c2e4682d4525a8d39e689bc39a86, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/389e5c23acd041599df719e4d4d0c6b8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/00fda2a08a82447d8a69f76fea7157c7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/a12b6dd451a0466b8a72107c04b4e9b4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ec6c00be327341b8a3d36b8c8d49dfa2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/43e8c57a06ec44cf95aa367dd7f86ae0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/809a656d760340b49fcb336f2c5f148a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ce5f9f339e704d0ebea72d911146b212, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/08273bf5bdfa4e80a6b9b112e9576399, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/f6332c6a4c1e4e648eea6ed40d62e0a5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/aa48171bd82f49b1a35720ab1f40ca74, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/adcc8e0d3105415696eeb12294bdad04, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/1eada155cc924cc299abc4392732575f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/eccd246a346641c5bdeff3a31c78983e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c418925d5c3a4bc082305c80409e723b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/33af8fff881e4300a17003c823010909, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/622a25f50cb444c68df4cd2e249d35b8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/88328216d22444c692570b7b28653c2a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/7165293846af442ea938c3ff8dc13878, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/b0c272acdaae402e85961bc9b151afed, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5a65994ab300449f9215725cb40627ff, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5cbc746d20f04ef89ba03a5c6af38765, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/3680a189140348599362ad6545872681, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/84dc50c52d20468187d474157cdc2462] to archive 2024-11-25T03:16:25,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:16:25,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/06f94fb0b5f24011b850ad64414ec877 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/06f94fb0b5f24011b850ad64414ec877 2024-11-25T03:16:25,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c199688239df426dae54e97c709b8c51 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c199688239df426dae54e97c709b8c51 2024-11-25T03:16:25,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/15c1c2e4682d4525a8d39e689bc39a86 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/15c1c2e4682d4525a8d39e689bc39a86 2024-11-25T03:16:25,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/389e5c23acd041599df719e4d4d0c6b8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/389e5c23acd041599df719e4d4d0c6b8 2024-11-25T03:16:25,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/00fda2a08a82447d8a69f76fea7157c7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/00fda2a08a82447d8a69f76fea7157c7 2024-11-25T03:16:25,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/a12b6dd451a0466b8a72107c04b4e9b4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/a12b6dd451a0466b8a72107c04b4e9b4 2024-11-25T03:16:25,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ec6c00be327341b8a3d36b8c8d49dfa2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ec6c00be327341b8a3d36b8c8d49dfa2 2024-11-25T03:16:25,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/43e8c57a06ec44cf95aa367dd7f86ae0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/43e8c57a06ec44cf95aa367dd7f86ae0 2024-11-25T03:16:25,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/809a656d760340b49fcb336f2c5f148a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/809a656d760340b49fcb336f2c5f148a 2024-11-25T03:16:25,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ce5f9f339e704d0ebea72d911146b212 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/ce5f9f339e704d0ebea72d911146b212 2024-11-25T03:16:25,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/08273bf5bdfa4e80a6b9b112e9576399 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/08273bf5bdfa4e80a6b9b112e9576399 2024-11-25T03:16:25,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/f6332c6a4c1e4e648eea6ed40d62e0a5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/f6332c6a4c1e4e648eea6ed40d62e0a5 2024-11-25T03:16:25,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/aa48171bd82f49b1a35720ab1f40ca74 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/aa48171bd82f49b1a35720ab1f40ca74 2024-11-25T03:16:25,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/adcc8e0d3105415696eeb12294bdad04 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/adcc8e0d3105415696eeb12294bdad04 2024-11-25T03:16:25,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/1eada155cc924cc299abc4392732575f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/1eada155cc924cc299abc4392732575f 2024-11-25T03:16:25,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/eccd246a346641c5bdeff3a31c78983e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/eccd246a346641c5bdeff3a31c78983e 2024-11-25T03:16:25,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c418925d5c3a4bc082305c80409e723b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/c418925d5c3a4bc082305c80409e723b 2024-11-25T03:16:25,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/33af8fff881e4300a17003c823010909 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/33af8fff881e4300a17003c823010909 2024-11-25T03:16:25,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/622a25f50cb444c68df4cd2e249d35b8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/622a25f50cb444c68df4cd2e249d35b8 2024-11-25T03:16:25,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/88328216d22444c692570b7b28653c2a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/88328216d22444c692570b7b28653c2a 2024-11-25T03:16:25,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/7165293846af442ea938c3ff8dc13878 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/7165293846af442ea938c3ff8dc13878 2024-11-25T03:16:25,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/b0c272acdaae402e85961bc9b151afed to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/b0c272acdaae402e85961bc9b151afed 2024-11-25T03:16:25,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5a65994ab300449f9215725cb40627ff to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5a65994ab300449f9215725cb40627ff 2024-11-25T03:16:25,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5cbc746d20f04ef89ba03a5c6af38765 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/5cbc746d20f04ef89ba03a5c6af38765 2024-11-25T03:16:25,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/3680a189140348599362ad6545872681 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/3680a189140348599362ad6545872681 2024-11-25T03:16:25,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/84dc50c52d20468187d474157cdc2462 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/84dc50c52d20468187d474157cdc2462 2024-11-25T03:16:25,801 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/recovered.edits/525.seqid, newMaxSeqId=525, maxSeqId=1 2024-11-25T03:16:25,802 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. 2024-11-25T03:16:25,802 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for d74b21d2d7105b5e1e755662f34cb2a2: Waiting for close lock at 1732504584493Running coprocessor pre-close hooks at 1732504584493Disabling compacts and flushes for region at 1732504584493Disabling writes for close at 1732504584493Obtaining lock to block concurrent updates at 1732504584493Preparing flush snapshotting stores in d74b21d2d7105b5e1e755662f34cb2a2 at 1732504584493Finished memstore snapshotting TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732504584493Flushing stores of TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2. at 1732504584494 (+1 ms)Flushing d74b21d2d7105b5e1e755662f34cb2a2/A: creating writer at 1732504584494Flushing d74b21d2d7105b5e1e755662f34cb2a2/A: appending metadata at 1732504584496 (+2 ms)Flushing d74b21d2d7105b5e1e755662f34cb2a2/A: closing flushed file at 1732504584496Flushing d74b21d2d7105b5e1e755662f34cb2a2/B: creating writer at 1732504584904 (+408 ms)Flushing d74b21d2d7105b5e1e755662f34cb2a2/B: appending metadata at 1732504584906 (+2 ms)Flushing d74b21d2d7105b5e1e755662f34cb2a2/B: closing flushed file at 1732504584906Flushing d74b21d2d7105b5e1e755662f34cb2a2/C: creating writer at 1732504585313 (+407 ms)Flushing d74b21d2d7105b5e1e755662f34cb2a2/C: appending metadata at 1732504585315 (+2 ms)Flushing d74b21d2d7105b5e1e755662f34cb2a2/C: closing flushed file at 1732504585315Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@380e1c60: reopening flushed file at 1732504585722 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cdb87a0: reopening flushed file at 1732504585726 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bd97e9: reopening flushed file at 1732504585730 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d74b21d2d7105b5e1e755662f34cb2a2 in 1240ms, sequenceid=522, compaction requested=true at 1732504585733 (+3 ms)Writing region close event to WAL at 1732504585799 (+66 ms)Running coprocessor post-close hooks at 1732504585802 (+3 ms)Closed at 1732504585802 2024-11-25T03:16:25,804 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:25,804 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d74b21d2d7105b5e1e755662f34cb2a2, regionState=CLOSED 2024-11-25T03:16:25,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:16:25,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-25T03:16:25,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure d74b21d2d7105b5e1e755662f34cb2a2, server=38a5d2077268,43357,1732504425108 in 1.4650 sec 2024-11-25T03:16:25,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-25T03:16:25,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d74b21d2d7105b5e1e755662f34cb2a2, UNASSIGN in 1.4690 sec 2024-11-25T03:16:25,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-25T03:16:25,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4720 sec 2024-11-25T03:16:25,811 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504585811"}]},"ts":"1732504585811"} 2024-11-25T03:16:25,812 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:16:25,812 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:16:25,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-25T03:16:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-25T03:16:26,490 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:26,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:16:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:26,492 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-25T03:16:26,493 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:26,495 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:26,496 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/recovered.edits] 2024-11-25T03:16:26,498 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/0de62ce8aeb1475191871e856eb9246e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/0de62ce8aeb1475191871e856eb9246e 2024-11-25T03:16:26,499 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/206ba4d0a96f4e038a3082d23ad5126f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/206ba4d0a96f4e038a3082d23ad5126f 2024-11-25T03:16:26,500 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/280c9961d05245f7b0961138cdf22a26 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/A/280c9961d05245f7b0961138cdf22a26 2024-11-25T03:16:26,502 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/497d15c7b14d496dbf93178edf3f3897 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/497d15c7b14d496dbf93178edf3f3897 2024-11-25T03:16:26,503 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ab6ec3aa2d8646099561c6282fd2032e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ab6ec3aa2d8646099561c6282fd2032e 2024-11-25T03:16:26,504 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ef94fd1cf66343968e31fa7944694280 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/B/ef94fd1cf66343968e31fa7944694280 2024-11-25T03:16:26,506 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/0383604f0d6e4a399ab4fa09570fb4cd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/0383604f0d6e4a399ab4fa09570fb4cd 2024-11-25T03:16:26,506 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/306dee7db24d4d1bace24c2bc9d4be58 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/306dee7db24d4d1bace24c2bc9d4be58 2024-11-25T03:16:26,507 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/e6138479e6864702b3b9aad483577bfb to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/C/e6138479e6864702b3b9aad483577bfb 2024-11-25T03:16:26,510 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/recovered.edits/525.seqid to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2/recovered.edits/525.seqid 2024-11-25T03:16:26,510 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/d74b21d2d7105b5e1e755662f34cb2a2 2024-11-25T03:16:26,510 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:16:26,512 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:26,514 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:16:26,516 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:16:26,516 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:26,517 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:16:26,517 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732504586517"}]},"ts":"9223372036854775807"} 2024-11-25T03:16:26,519 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:16:26,519 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d74b21d2d7105b5e1e755662f34cb2a2, NAME => 'TestAcidGuarantees,,1732504558668.d74b21d2d7105b5e1e755662f34cb2a2.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:16:26,519 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:16:26,519 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732504586519"}]},"ts":"9223372036854775807"} 2024-11-25T03:16:26,520 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:16:26,521 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:26,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-25T03:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-25T03:16:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:16:26,600 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:26,609 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=382 (was 365) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6273 (was 6375) 2024-11-25T03:16:26,618 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=382, ProcessCount=11, AvailableMemoryMB=6273 2024-11-25T03:16:26,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:16:26,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:26,621 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:16:26,621 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:26,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-25T03:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:16:26,622 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:16:26,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742333_1509 (size=1061) 2024-11-25T03:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:16:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:16:27,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf 2024-11-25T03:16:27,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742334_1510 (size=53) 2024-11-25T03:16:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:16:27,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:16:27,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 08a856257124e7cc799bf8e479f21703, disabling compactions & flushes 2024-11-25T03:16:27,434 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. after waiting 0 ms 2024-11-25T03:16:27,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,434 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 08a856257124e7cc799bf8e479f21703: Waiting for close lock at 1732504587434Disabling compacts and flushes for region at 1732504587434Disabling writes for close at 1732504587434Writing region close event to WAL at 1732504587434Closed at 1732504587434 2024-11-25T03:16:27,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:16:27,435 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732504587435"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732504587435"}]},"ts":"1732504587435"} 2024-11-25T03:16:27,436 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:16:27,437 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:16:27,437 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504587437"}]},"ts":"1732504587437"} 2024-11-25T03:16:27,438 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:16:27,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, ASSIGN}] 2024-11-25T03:16:27,439 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, ASSIGN 2024-11-25T03:16:27,439 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, ASSIGN; state=OFFLINE, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=false 2024-11-25T03:16:27,590 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:16:27,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, ASSIGN because future has completed 2024-11-25T03:16:27,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:16:27,746 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,746 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:16:27,746 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,746 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:16:27,746 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,746 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,747 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,748 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:16:27,748 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08a856257124e7cc799bf8e479f21703 columnFamilyName A 2024-11-25T03:16:27,748 DEBUG [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:27,749 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(327): Store=08a856257124e7cc799bf8e479f21703/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:16:27,749 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,750 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:16:27,750 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08a856257124e7cc799bf8e479f21703 columnFamilyName B 2024-11-25T03:16:27,750 DEBUG [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:27,750 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(327): Store=08a856257124e7cc799bf8e479f21703/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:16:27,750 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,751 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:16:27,751 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08a856257124e7cc799bf8e479f21703 columnFamilyName C 2024-11-25T03:16:27,751 DEBUG [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:27,751 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(327): Store=08a856257124e7cc799bf8e479f21703/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:16:27,751 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,751 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,752 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,752 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,753 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,753 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,753 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:16:27,754 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,756 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:16:27,756 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened 08a856257124e7cc799bf8e479f21703; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65125337, jitterRate=-0.02955685555934906}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:16:27,756 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:27,757 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for 08a856257124e7cc799bf8e479f21703: Running coprocessor pre-open hook at 1732504587746Writing region info on filesystem at 1732504587746Initializing all the Stores at 1732504587747 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504587747Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504587747Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504587747Cleaning up temporary data from old regions at 1732504587753 (+6 ms)Running coprocessor post-open hooks at 1732504587756 (+3 ms)Region opened successfully at 1732504587757 (+1 ms) 2024-11-25T03:16:27,757 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., pid=150, masterSystemTime=1732504587743 2024-11-25T03:16:27,759 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,759 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:27,759 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:16:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:16:27,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:16:27,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-25T03:16:27,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 in 169 msec 2024-11-25T03:16:27,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-25T03:16:27,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, ASSIGN in 324 msec 2024-11-25T03:16:27,764 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:16:27,765 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504587764"}]},"ts":"1732504587764"} 2024-11-25T03:16:27,765 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:16:27,766 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:16:27,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-25T03:16:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:16:28,770 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:28,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:28,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:28,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:28,772 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540785ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:28,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df3f462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:28,775 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:28,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:28,777 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:16:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,44623,1732504424313 2024-11-25T03:16:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a31a2ac 2024-11-25T03:16:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:16:28,778 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:16:28,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:16:28,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:16:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T03:16:28,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742335_1511 (size=1097) 2024-11-25T03:16:29,190 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T03:16:29,190 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T03:16:29,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:16:29,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, REOPEN/MOVE}] 2024-11-25T03:16:29,195 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, REOPEN/MOVE 2024-11-25T03:16:29,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, REOPEN/MOVE because future has completed 2024-11-25T03:16:29,197 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:16:29,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:16:29,349 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,349 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:16:29,349 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing 08a856257124e7cc799bf8e479f21703, disabling compactions & flushes 2024-11-25T03:16:29,349 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,349 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,349 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. after waiting 0 ms 2024-11-25T03:16:29,349 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,352 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:16:29,353 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,353 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for 08a856257124e7cc799bf8e479f21703: Waiting for close lock at 1732504589349Running coprocessor pre-close hooks at 1732504589349Disabling compacts and flushes for region at 1732504589349Disabling writes for close at 1732504589349Writing region close event to WAL at 1732504589349Running coprocessor post-close hooks at 1732504589353 (+4 ms)Closed at 1732504589353 2024-11-25T03:16:29,353 WARN [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: 08a856257124e7cc799bf8e479f21703 to self. 2024-11-25T03:16:29,354 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,355 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=CLOSED 2024-11-25T03:16:29,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:16:29,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-25T03:16:29,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 in 160 msec 2024-11-25T03:16:29,358 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, REOPEN/MOVE; state=CLOSED, location=38a5d2077268,43357,1732504425108; forceNewPlan=false, retain=true 2024-11-25T03:16:29,509 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=OPENING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, REOPEN/MOVE because future has completed 2024-11-25T03:16:29,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:16:29,664 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,664 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:16:29,664 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,665 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:16:29,665 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,665 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,666 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,666 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:16:29,666 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08a856257124e7cc799bf8e479f21703 columnFamilyName A 2024-11-25T03:16:29,667 DEBUG [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:29,668 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(327): Store=08a856257124e7cc799bf8e479f21703/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:16:29,668 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,669 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:16:29,669 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08a856257124e7cc799bf8e479f21703 columnFamilyName B 2024-11-25T03:16:29,669 DEBUG [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:29,669 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(327): Store=08a856257124e7cc799bf8e479f21703/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:16:29,669 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,670 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T03:16:29,670 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08a856257124e7cc799bf8e479f21703 columnFamilyName C 2024-11-25T03:16:29,670 DEBUG [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:29,670 INFO [StoreOpener-08a856257124e7cc799bf8e479f21703-1 {}] regionserver.HStore(327): Store=08a856257124e7cc799bf8e479f21703/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:16:29,670 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,670 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,671 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,671 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,672 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,672 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,672 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:16:29,673 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,674 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened 08a856257124e7cc799bf8e479f21703; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59635725, jitterRate=-0.11135844886302948}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:16:29,674 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,674 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for 08a856257124e7cc799bf8e479f21703: Running coprocessor pre-open hook at 1732504589665Writing region info on filesystem at 1732504589665Initializing all the Stores at 1732504589665Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504589665Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504589666 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732504589666Cleaning up temporary data from old regions at 1732504589672 (+6 ms)Running coprocessor post-open hooks at 1732504589674 (+2 ms)Region opened successfully at 1732504589674 2024-11-25T03:16:29,675 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., pid=155, masterSystemTime=1732504589662 2024-11-25T03:16:29,676 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,676 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,677 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=OPEN, openSeqNum=5, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:16:29,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-25T03:16:29,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 in 168 msec 2024-11-25T03:16:29,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-25T03:16:29,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, REOPEN/MOVE in 486 msec 2024-11-25T03:16:29,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-25T03:16:29,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-25T03:16:29,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-25T03:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-25T03:16:29,685 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:29,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb36afd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6defefe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e55aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38318c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba0627f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d22f9f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,694 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c73a84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722e5389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52998492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,698 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319a099f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,700 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392ff92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@133e623d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b36a10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,704 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a02b22c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,705 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505780b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,706 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@766ead7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6577b8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251b2363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,44623,-1 for getting cluster id 2024-11-25T03:16:29,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:16:29,711 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8407abcc-f40b-4057-8321-91525263860d' 2024-11-25T03:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8407abcc-f40b-4057-8321-91525263860d" 2024-11-25T03:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6f473f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,44623,-1] 2024-11-25T03:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:29,712 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:16:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f8d995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:16:29,713 DEBUG [Thread-2300 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,713 DEBUG [Thread-2317 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,713 DEBUG [Thread-2315 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,713 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,713 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,714 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,714 DEBUG [Thread-2311 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:29,715 DEBUG [Thread-2313 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,715 DEBUG [Thread-2306 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,716 DEBUG [Thread-2319 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-25T03:16:29,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:16:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,716 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:16:29,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:29,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,43357,1732504425108, seqNum=-1] 2024-11-25T03:16:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:16:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:16:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5] 2024-11-25T03:16:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:29,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T03:16:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504649745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504589741, max timestamp=1732504589749Num uniques 859; 2024-11-25T03:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504589741, max timestamp=1732504589749Num uniques 859; 2024-11-25T03:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:29,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732504589741, max timestamp=1732504589749Num uniques 859; 2024-11-25T03:16:29,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T03:16:29,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125522fd138bfc34951a3c17d70d38460eb_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504589725/Put/seqid=0 2024-11-25T03:16:29,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742336_1512 (size=12154) 2024-11-25T03:16:29,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504589772, max timestamp=1732504589781Num uniques 1018; 2024-11-25T03:16:29,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-25T03:16:29,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504589772, max timestamp=1732504589781Num uniques 1018; 2024-11-25T03:16:29,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-25T03:16:29,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732504589772, max timestamp=1732504589781Num uniques 1018; 2024-11-25T03:16:29,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/38a5d2077268:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-25T03:16:29,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504649807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504649809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504649810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504649812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:16:29,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504649859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:29,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:29,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504649920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504649920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504649920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504649921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:16:30,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504650070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504650129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504650130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504650130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504650130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,178 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:30,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125522fd138bfc34951a3c17d70d38460eb_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125522fd138bfc34951a3c17d70d38460eb_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:30,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/3add0679b0784ba0994dd5b3ab2cdcb2, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:30,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/3add0679b0784ba0994dd5b3ab2cdcb2 is 175, key is test_row_0/A:col10/1732504589725/Put/seqid=0 2024-11-25T03:16:30,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742337_1513 (size=30955) 2024-11-25T03:16:30,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:16:30,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504650380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504650439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504650440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504650440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504650440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,601 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/3add0679b0784ba0994dd5b3ab2cdcb2 2024-11-25T03:16:30,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3 is 50, key is test_row_0/B:col10/1732504589725/Put/seqid=0 2024-11-25T03:16:30,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742338_1514 (size=12001) 2024-11-25T03:16:30,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3 2024-11-25T03:16:30,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/d151f040a02e4cbfa43e5e5b8ae8052d is 50, key is test_row_0/C:col10/1732504589725/Put/seqid=0 2024-11-25T03:16:30,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742339_1515 (size=12001) 2024-11-25T03:16:30,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:16:30,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504650890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:30,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504650950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504650950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504650950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504650950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:31,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/d151f040a02e4cbfa43e5e5b8ae8052d 2024-11-25T03:16:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/3add0679b0784ba0994dd5b3ab2cdcb2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3add0679b0784ba0994dd5b3ab2cdcb2 2024-11-25T03:16:31,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3add0679b0784ba0994dd5b3ab2cdcb2, entries=150, sequenceid=15, filesize=30.2 K 2024-11-25T03:16:31,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3 2024-11-25T03:16:31,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:16:31,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/d151f040a02e4cbfa43e5e5b8ae8052d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d151f040a02e4cbfa43e5e5b8ae8052d 2024-11-25T03:16:31,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d151f040a02e4cbfa43e5e5b8ae8052d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:16:31,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 08a856257124e7cc799bf8e479f21703 in 1351ms, sequenceid=15, compaction requested=false 2024-11-25T03:16:31,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:31,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:16:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:31,095 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=127.47 KB heapSize=359.53 KB 2024-11-25T03:16:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T03:16:31,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ca0e7c81d4014f7c90a8b73cddf19060_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504589807/Put/seqid=0 2024-11-25T03:16:31,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742340_1516 (size=12154) 2024-11-25T03:16:31,322 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:16:31,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:31,511 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ca0e7c81d4014f7c90a8b73cddf19060_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ca0e7c81d4014f7c90a8b73cddf19060_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:31,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/6f2e071549bf47ce91fd1919d9a04a99, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:31,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/6f2e071549bf47ce91fd1919d9a04a99 is 175, key is test_row_0/A:col10/1732504589807/Put/seqid=0 2024-11-25T03:16:31,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742341_1517 (size=30955) 2024-11-25T03:16:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:16:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:31,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:31,916 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=67, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/6f2e071549bf47ce91fd1919d9a04a99 2024-11-25T03:16:31,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/cfad0a23620a48979d988da12e54f558 is 50, key is test_row_0/B:col10/1732504589807/Put/seqid=0 2024-11-25T03:16:31,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742342_1518 (size=12001) 2024-11-25T03:16:31,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504651960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:31,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504651960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:31,964 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:31,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:31,964 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504651963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:31,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504651969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:31,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504651969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:31,974 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:31,974 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:32,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504652070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504652280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:32,331 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/cfad0a23620a48979d988da12e54f558 2024-11-25T03:16:32,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/bbc5657ea3334bec8d668cfffb92dd4c is 50, key is test_row_0/C:col10/1732504589807/Put/seqid=0 2024-11-25T03:16:32,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742343_1519 (size=12001) 2024-11-25T03:16:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504652590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:32,743 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/bbc5657ea3334bec8d668cfffb92dd4c 2024-11-25T03:16:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/6f2e071549bf47ce91fd1919d9a04a99 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/6f2e071549bf47ce91fd1919d9a04a99 2024-11-25T03:16:32,750 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/6f2e071549bf47ce91fd1919d9a04a99, entries=150, sequenceid=67, filesize=30.2 K 2024-11-25T03:16:32,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/cfad0a23620a48979d988da12e54f558 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cfad0a23620a48979d988da12e54f558 2024-11-25T03:16:32,755 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cfad0a23620a48979d988da12e54f558, entries=150, sequenceid=67, filesize=11.7 K 2024-11-25T03:16:32,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/bbc5657ea3334bec8d668cfffb92dd4c as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/bbc5657ea3334bec8d668cfffb92dd4c 2024-11-25T03:16:32,759 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/bbc5657ea3334bec8d668cfffb92dd4c, entries=150, sequenceid=67, filesize=11.7 K 2024-11-25T03:16:32,760 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~359.48 KB/368112, currentSize=53.67 KB/54960 for 08a856257124e7cc799bf8e479f21703 in 1665ms, sequenceid=67, compaction requested=false 2024-11-25T03:16:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-25T03:16:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-25T03:16:32,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-25T03:16:32,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0430 sec 2024-11-25T03:16:32,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.0480 sec 2024-11-25T03:16:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:33,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T03:16:33,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:33,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:33,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:33,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:33,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:33,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:33,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257ef2ccf5e02143d59c973de67dea1e74_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504593101/Put/seqid=0 2024-11-25T03:16:33,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742344_1520 (size=12154) 2024-11-25T03:16:33,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:33,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504653188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:33,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504653300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:33,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504653509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:33,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:33,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257ef2ccf5e02143d59c973de67dea1e74_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ef2ccf5e02143d59c973de67dea1e74_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:33,521 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/a332729d2c364ecf85fa053a49f1d90f, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:33,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/a332729d2c364ecf85fa053a49f1d90f is 175, key is test_row_0/A:col10/1732504593101/Put/seqid=0 2024-11-25T03:16:33,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742345_1521 (size=30955) 2024-11-25T03:16:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504653820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:16:33,880 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:33,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-25T03:16:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:16:33,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:33,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:33,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:33,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/a332729d2c364ecf85fa053a49f1d90f 2024-11-25T03:16:33,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/169dfc6cab2f4307a3b884a865bb3b7d is 50, key is test_row_0/B:col10/1732504593101/Put/seqid=0 2024-11-25T03:16:33,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742346_1522 (size=12001) 2024-11-25T03:16:33,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/169dfc6cab2f4307a3b884a865bb3b7d 2024-11-25T03:16:33,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8877f6afc7504c3fbc523223d5f4e670 is 50, key is test_row_0/C:col10/1732504593101/Put/seqid=0 2024-11-25T03:16:33,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742347_1523 (size=12001) 2024-11-25T03:16:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504653980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:33,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504653980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:33,984 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:33,984 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:16:33,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504653990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:33,995 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_0' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:34,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504654000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:34,003 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:34,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T03:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T03:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:16:34,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504654330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:34,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T03:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:34,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8877f6afc7504c3fbc523223d5f4e670 2024-11-25T03:16:34,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/a332729d2c364ecf85fa053a49f1d90f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/a332729d2c364ecf85fa053a49f1d90f 2024-11-25T03:16:34,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/a332729d2c364ecf85fa053a49f1d90f, entries=150, sequenceid=79, filesize=30.2 K 2024-11-25T03:16:34,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/169dfc6cab2f4307a3b884a865bb3b7d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/169dfc6cab2f4307a3b884a865bb3b7d 2024-11-25T03:16:34,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/169dfc6cab2f4307a3b884a865bb3b7d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T03:16:34,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8877f6afc7504c3fbc523223d5f4e670 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8877f6afc7504c3fbc523223d5f4e670 2024-11-25T03:16:34,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8877f6afc7504c3fbc523223d5f4e670, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T03:16:34,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 08a856257124e7cc799bf8e479f21703 in 1273ms, sequenceid=79, compaction requested=true 2024-11-25T03:16:34,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:34,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:34,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:34,377 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:34,377 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:34,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:34,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:34,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:34,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:34,380 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:34,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:34,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/A is initiating minor compaction (all files) 2024-11-25T03:16:34,380 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/B is initiating minor compaction (all files) 2024-11-25T03:16:34,380 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/B in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,380 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/A in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,380 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cfad0a23620a48979d988da12e54f558, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/169dfc6cab2f4307a3b884a865bb3b7d] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.2 K 2024-11-25T03:16:34,380 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3add0679b0784ba0994dd5b3ab2cdcb2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/6f2e071549bf47ce91fd1919d9a04a99, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/a332729d2c364ecf85fa053a49f1d90f] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=90.7 K 2024-11-25T03:16:34,380 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,380 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3add0679b0784ba0994dd5b3ab2cdcb2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/6f2e071549bf47ce91fd1919d9a04a99, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/a332729d2c364ecf85fa053a49f1d90f] 2024-11-25T03:16:34,380 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b73e1dac6bd4dfd9c2a2035cb9f4bf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732504589723 2024-11-25T03:16:34,381 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3add0679b0784ba0994dd5b3ab2cdcb2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732504589723 2024-11-25T03:16:34,381 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting cfad0a23620a48979d988da12e54f558, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732504589802 2024-11-25T03:16:34,381 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2e071549bf47ce91fd1919d9a04a99, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732504589802 2024-11-25T03:16:34,381 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting a332729d2c364ecf85fa053a49f1d90f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732504591949 2024-11-25T03:16:34,381 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 169dfc6cab2f4307a3b884a865bb3b7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732504591949 2024-11-25T03:16:34,386 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:34,388 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:34,388 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125164791e4168d4f56b54afe5a5f2d7af5_08a856257124e7cc799bf8e479f21703 store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:34,388 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/a21bea1770fc45a983bd2e8adb7f4bfe is 50, key is test_row_0/B:col10/1732504593101/Put/seqid=0 2024-11-25T03:16:34,390 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125164791e4168d4f56b54afe5a5f2d7af5_08a856257124e7cc799bf8e479f21703, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:34,390 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125164791e4168d4f56b54afe5a5f2d7af5_08a856257124e7cc799bf8e479f21703 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:34,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742348_1524 (size=12104) 2024-11-25T03:16:34,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742349_1525 (size=4469) 2024-11-25T03:16:34,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T03:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,496 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T03:16:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:34,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250126a6623e7342aab00c76c235da9caf_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504593170/Put/seqid=0 2024-11-25T03:16:34,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742350_1526 (size=12154) 2024-11-25T03:16:34,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:16:34,510 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250126a6623e7342aab00c76c235da9caf_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250126a6623e7342aab00c76c235da9caf_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fcc2f0a3461249bea0bb4fa361134c23, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fcc2f0a3461249bea0bb4fa361134c23 is 175, key is test_row_0/A:col10/1732504593170/Put/seqid=0 2024-11-25T03:16:34,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742351_1527 (size=30955) 2024-11-25T03:16:34,804 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#A#compaction#447 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:34,805 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/94b2f65d0ae848208944e144df694da1 is 175, key is test_row_0/A:col10/1732504593101/Put/seqid=0 2024-11-25T03:16:34,808 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/a21bea1770fc45a983bd2e8adb7f4bfe as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a21bea1770fc45a983bd2e8adb7f4bfe 2024-11-25T03:16:34,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742352_1528 (size=31058) 2024-11-25T03:16:34,813 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/B of 08a856257124e7cc799bf8e479f21703 into a21bea1770fc45a983bd2e8adb7f4bfe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:34,813 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:34,813 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/B, priority=13, startTime=1732504594377; duration=0sec 2024-11-25T03:16:34,814 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:34,814 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:B 2024-11-25T03:16:34,814 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:34,814 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:34,815 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/C is initiating minor compaction (all files) 2024-11-25T03:16:34,815 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/C in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:34,815 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d151f040a02e4cbfa43e5e5b8ae8052d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/bbc5657ea3334bec8d668cfffb92dd4c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8877f6afc7504c3fbc523223d5f4e670] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.2 K 2024-11-25T03:16:34,815 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d151f040a02e4cbfa43e5e5b8ae8052d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732504589723 2024-11-25T03:16:34,815 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc5657ea3334bec8d668cfffb92dd4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732504589802 2024-11-25T03:16:34,815 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8877f6afc7504c3fbc523223d5f4e670, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732504591949 2024-11-25T03:16:34,822 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:34,822 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/575fe36447d14176a8b119306fab35ab is 50, key is test_row_0/C:col10/1732504593101/Put/seqid=0 2024-11-25T03:16:34,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742353_1529 (size=12104) 2024-11-25T03:16:34,829 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/575fe36447d14176a8b119306fab35ab as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/575fe36447d14176a8b119306fab35ab 2024-11-25T03:16:34,833 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/C of 08a856257124e7cc799bf8e479f21703 into 575fe36447d14176a8b119306fab35ab(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:34,833 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:34,833 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/C, priority=13, startTime=1732504594377; duration=0sec 2024-11-25T03:16:34,833 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:34,833 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:C 2024-11-25T03:16:34,916 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fcc2f0a3461249bea0bb4fa361134c23 2024-11-25T03:16:34,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/18e660c3b898475da74683207416eb2a is 50, key is test_row_0/B:col10/1732504593170/Put/seqid=0 2024-11-25T03:16:34,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742354_1530 (size=12001) 2024-11-25T03:16:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:16:35,217 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/94b2f65d0ae848208944e144df694da1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/94b2f65d0ae848208944e144df694da1 2024-11-25T03:16:35,221 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/A of 08a856257124e7cc799bf8e479f21703 into 94b2f65d0ae848208944e144df694da1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:35,221 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:35,221 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/A, priority=13, startTime=1732504594377; duration=0sec 2024-11-25T03:16:35,221 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:35,221 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:A 2024-11-25T03:16:35,326 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/18e660c3b898475da74683207416eb2a 2024-11-25T03:16:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/2404b98c297c4b23a50a58a8c6eb6929 is 50, key is test_row_0/C:col10/1732504593170/Put/seqid=0 2024-11-25T03:16:35,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742355_1531 (size=12001) 2024-11-25T03:16:35,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:35,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504655381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:35,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504655490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:35,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504655700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:35,737 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/2404b98c297c4b23a50a58a8c6eb6929 2024-11-25T03:16:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fcc2f0a3461249bea0bb4fa361134c23 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fcc2f0a3461249bea0bb4fa361134c23 2024-11-25T03:16:35,745 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fcc2f0a3461249bea0bb4fa361134c23, entries=150, sequenceid=99, filesize=30.2 K 2024-11-25T03:16:35,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/18e660c3b898475da74683207416eb2a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/18e660c3b898475da74683207416eb2a 2024-11-25T03:16:35,749 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/18e660c3b898475da74683207416eb2a, entries=150, sequenceid=99, filesize=11.7 K 2024-11-25T03:16:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/2404b98c297c4b23a50a58a8c6eb6929 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/2404b98c297c4b23a50a58a8c6eb6929 2024-11-25T03:16:35,753 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/2404b98c297c4b23a50a58a8c6eb6929, entries=150, sequenceid=99, filesize=11.7 K 2024-11-25T03:16:35,754 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 08a856257124e7cc799bf8e479f21703 in 1259ms, sequenceid=99, compaction requested=false 2024-11-25T03:16:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-25T03:16:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-25T03:16:35,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-25T03:16:35,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8720 sec 2024-11-25T03:16:35,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8770 sec 2024-11-25T03:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:36,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T03:16:36,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:36,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:36,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:36,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:36,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:36,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:36,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112505083e88d04d4711b38e75bdecfc778c_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504595350/Put/seqid=0 2024-11-25T03:16:36,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742356_1532 (size=14594) 2024-11-25T03:16:36,027 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:36,030 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112505083e88d04d4711b38e75bdecfc778c_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112505083e88d04d4711b38e75bdecfc778c_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:16:36,031 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:36,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-25T03:16:36,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/91d85bdef0c04ba2ab2e834bcbe0f91a, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:36,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/91d85bdef0c04ba2ab2e834bcbe0f91a is 175, key is test_row_0/A:col10/1732504595350/Put/seqid=0 2024-11-25T03:16:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:16:36,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:36,034 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:36,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:36,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742357_1533 (size=39549) 2024-11-25T03:16:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504656081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:16:36,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504656190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:36,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:16:36,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504656400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:36,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/91d85bdef0c04ba2ab2e834bcbe0f91a 2024-11-25T03:16:36,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/e281a111bb0f4391b795e6e32f6f92fa is 50, key is test_row_0/B:col10/1732504595350/Put/seqid=0 2024-11-25T03:16:36,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742358_1534 (size=12001) 2024-11-25T03:16:36,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:36,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:36,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:16:36,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504656710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:36,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/e281a111bb0f4391b795e6e32f6f92fa 2024-11-25T03:16:36,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/02d8a8de5348477b91fb4676b8fb7c6e is 50, key is test_row_0/C:col10/1732504595350/Put/seqid=0 2024-11-25T03:16:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742359_1535 (size=12001) 2024-11-25T03:16:36,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:36,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:36,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:37,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:37,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:37,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:16:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504657220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:37,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:37,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:37,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/02d8a8de5348477b91fb4676b8fb7c6e 2024-11-25T03:16:37,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/91d85bdef0c04ba2ab2e834bcbe0f91a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/91d85bdef0c04ba2ab2e834bcbe0f91a 2024-11-25T03:16:37,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/91d85bdef0c04ba2ab2e834bcbe0f91a, entries=200, sequenceid=115, filesize=38.6 K 2024-11-25T03:16:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/e281a111bb0f4391b795e6e32f6f92fa as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/e281a111bb0f4391b795e6e32f6f92fa 2024-11-25T03:16:37,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/e281a111bb0f4391b795e6e32f6f92fa, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T03:16:37,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/02d8a8de5348477b91fb4676b8fb7c6e as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/02d8a8de5348477b91fb4676b8fb7c6e 2024-11-25T03:16:37,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/02d8a8de5348477b91fb4676b8fb7c6e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T03:16:37,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 08a856257124e7cc799bf8e479f21703 in 1268ms, sequenceid=115, compaction requested=true 2024-11-25T03:16:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:37,281 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:37,281 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:37,282 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:37,282 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:37,282 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/B is initiating minor compaction (all files) 2024-11-25T03:16:37,282 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/A is initiating minor compaction (all files) 2024-11-25T03:16:37,282 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/A in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,282 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/B in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,282 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a21bea1770fc45a983bd2e8adb7f4bfe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/18e660c3b898475da74683207416eb2a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/e281a111bb0f4391b795e6e32f6f92fa] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.3 K 2024-11-25T03:16:37,282 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/94b2f65d0ae848208944e144df694da1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fcc2f0a3461249bea0bb4fa361134c23, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/91d85bdef0c04ba2ab2e834bcbe0f91a] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=99.2 K 2024-11-25T03:16:37,282 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,282 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/94b2f65d0ae848208944e144df694da1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fcc2f0a3461249bea0bb4fa361134c23, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/91d85bdef0c04ba2ab2e834bcbe0f91a] 2024-11-25T03:16:37,283 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a21bea1770fc45a983bd2e8adb7f4bfe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732504591949 2024-11-25T03:16:37,283 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b2f65d0ae848208944e144df694da1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732504591949 2024-11-25T03:16:37,283 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 18e660c3b898475da74683207416eb2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732504593170 2024-11-25T03:16:37,283 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc2f0a3461249bea0bb4fa361134c23, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732504593170 2024-11-25T03:16:37,283 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting e281a111bb0f4391b795e6e32f6f92fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504595350 2024-11-25T03:16:37,284 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d85bdef0c04ba2ab2e834bcbe0f91a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504595350 2024-11-25T03:16:37,288 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:37,291 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:37,291 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/0fe55677b719460ba4db407dd57d9b59 is 50, key is test_row_0/B:col10/1732504595350/Put/seqid=0 2024-11-25T03:16:37,293 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112516931ad5020a442cb2c97a9369a162f5_08a856257124e7cc799bf8e479f21703 store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:37,294 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112516931ad5020a442cb2c97a9369a162f5_08a856257124e7cc799bf8e479f21703, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:37,294 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112516931ad5020a442cb2c97a9369a162f5_08a856257124e7cc799bf8e479f21703 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:37,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742360_1536 (size=12207) 2024-11-25T03:16:37,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742361_1537 (size=4469) 2024-11-25T03:16:37,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:16:37,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,409 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T03:16:37,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:37,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:37,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:37,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:37,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:37,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:37,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112580c35cffad9041268d0cd3e7029a7089_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504596069/Put/seqid=0 2024-11-25T03:16:37,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742362_1538 (size=12254) 2024-11-25T03:16:37,702 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/0fe55677b719460ba4db407dd57d9b59 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0fe55677b719460ba4db407dd57d9b59 2024-11-25T03:16:37,706 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/B of 08a856257124e7cc799bf8e479f21703 into 0fe55677b719460ba4db407dd57d9b59(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:37,706 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:37,706 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/B, priority=13, startTime=1732504597281; duration=0sec 2024-11-25T03:16:37,706 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:37,706 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:B 2024-11-25T03:16:37,706 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:37,707 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:37,707 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#A#compaction#456 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:37,707 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/C is initiating minor compaction (all files) 2024-11-25T03:16:37,707 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/C in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:37,708 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/575fe36447d14176a8b119306fab35ab, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/2404b98c297c4b23a50a58a8c6eb6929, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/02d8a8de5348477b91fb4676b8fb7c6e] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.3 K 2024-11-25T03:16:37,708 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/dbde371f0cb6410d8413d0aa462bf1bd is 175, key is test_row_0/A:col10/1732504595350/Put/seqid=0 2024-11-25T03:16:37,708 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 575fe36447d14176a8b119306fab35ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732504591949 2024-11-25T03:16:37,709 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 2404b98c297c4b23a50a58a8c6eb6929, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732504593170 2024-11-25T03:16:37,710 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d8a8de5348477b91fb4676b8fb7c6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504595350 2024-11-25T03:16:37,730 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:37,731 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/55932c66fc6e4a10a142d257dfd903b4 is 50, key is test_row_0/C:col10/1732504595350/Put/seqid=0 2024-11-25T03:16:37,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742363_1539 (size=31161) 2024-11-25T03:16:37,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742364_1540 (size=12207) 2024-11-25T03:16:37,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:37,824 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112580c35cffad9041268d0cd3e7029a7089_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112580c35cffad9041268d0cd3e7029a7089_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:37,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b54664de9cef4babaeaad57f4fe4a2db, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:37,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b54664de9cef4babaeaad57f4fe4a2db is 175, key is test_row_0/A:col10/1732504596069/Put/seqid=0 2024-11-25T03:16:37,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742365_1541 (size=31055) 2024-11-25T03:16:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:38,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:38,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504658021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504658022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504658024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504658030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,034 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504658129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504658129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,138 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/dbde371f0cb6410d8413d0aa462bf1bd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/dbde371f0cb6410d8413d0aa462bf1bd 2024-11-25T03:16:38,140 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/55932c66fc6e4a10a142d257dfd903b4 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/55932c66fc6e4a10a142d257dfd903b4 2024-11-25T03:16:38,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504658140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,143 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/A of 08a856257124e7cc799bf8e479f21703 into dbde371f0cb6410d8413d0aa462bf1bd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:38,143 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:38,143 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/A, priority=13, startTime=1732504597281; duration=0sec 2024-11-25T03:16:38,143 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:38,143 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:A 2024-11-25T03:16:38,144 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/C of 08a856257124e7cc799bf8e479f21703 into 55932c66fc6e4a10a142d257dfd903b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:38,144 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:38,144 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/C, priority=13, startTime=1732504597281; duration=0sec 2024-11-25T03:16:38,144 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:38,144 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:C 2024-11-25T03:16:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:16:38,230 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b54664de9cef4babaeaad57f4fe4a2db 2024-11-25T03:16:38,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/d7ae1e727d7c454283a0ecc0b3c57971 is 50, key is test_row_0/B:col10/1732504596069/Put/seqid=0 2024-11-25T03:16:38,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742366_1542 (size=12101) 2024-11-25T03:16:38,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504658240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,243 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504658340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504658340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504658350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,641 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/d7ae1e727d7c454283a0ecc0b3c57971 2024-11-25T03:16:38,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/f0cca49c2ba043119461dbdb8a4385c1 is 50, key is test_row_0/C:col10/1732504596069/Put/seqid=0 2024-11-25T03:16:38,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504658650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504658650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:38,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742367_1543 (size=12101) 2024-11-25T03:16:38,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504658660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,057 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/f0cca49c2ba043119461dbdb8a4385c1 2024-11-25T03:16:39,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b54664de9cef4babaeaad57f4fe4a2db as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b54664de9cef4babaeaad57f4fe4a2db 2024-11-25T03:16:39,066 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b54664de9cef4babaeaad57f4fe4a2db, entries=150, sequenceid=134, filesize=30.3 K 2024-11-25T03:16:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/d7ae1e727d7c454283a0ecc0b3c57971 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/d7ae1e727d7c454283a0ecc0b3c57971 2024-11-25T03:16:39,069 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/d7ae1e727d7c454283a0ecc0b3c57971, entries=150, sequenceid=134, filesize=11.8 K 2024-11-25T03:16:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/f0cca49c2ba043119461dbdb8a4385c1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/f0cca49c2ba043119461dbdb8a4385c1 2024-11-25T03:16:39,073 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/f0cca49c2ba043119461dbdb8a4385c1, entries=150, sequenceid=134, filesize=11.8 K 2024-11-25T03:16:39,074 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 08a856257124e7cc799bf8e479f21703 in 1665ms, sequenceid=134, compaction requested=false 2024-11-25T03:16:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-25T03:16:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-25T03:16:39,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-25T03:16:39,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0410 sec 2024-11-25T03:16:39,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.0460 sec 2024-11-25T03:16:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:39,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T03:16:39,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:39,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:39,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:39,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:39,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:39,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:39,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112561d6ce901103492b85cb3363f043fed4_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504598019/Put/seqid=0 2024-11-25T03:16:39,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742368_1544 (size=14794) 2024-11-25T03:16:39,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504659184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504659186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504659189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504659301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504659302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504659302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504659510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504659510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504659510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:39,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112561d6ce901103492b85cb3363f043fed4_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112561d6ce901103492b85cb3363f043fed4_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:39,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fb6b69c2772849cb8dcde88d60318c20, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:39,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fb6b69c2772849cb8dcde88d60318c20 is 175, key is test_row_0/A:col10/1732504598019/Put/seqid=0 2024-11-25T03:16:39,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742369_1545 (size=39749) 2024-11-25T03:16:39,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504659820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504659820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504659820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:39,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fb6b69c2772849cb8dcde88d60318c20 2024-11-25T03:16:39,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/8fdf59517bbe4cdd9ce679cb21da5241 is 50, key is test_row_0/B:col10/1732504598019/Put/seqid=0 2024-11-25T03:16:39,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742370_1546 (size=12151) 2024-11-25T03:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:16:40,201 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:40,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-25T03:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:16:40,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:40,204 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:40,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:40,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504660250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:40,253 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:16:40,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504660330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:40,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504660330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:40,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504660330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:40,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:40,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:40,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-25T03:16:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/8fdf59517bbe4cdd9ce679cb21da5241 2024-11-25T03:16:40,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/73f15b71719f45038a856884ab5ada3d is 50, key is test_row_0/C:col10/1732504598019/Put/seqid=0 2024-11-25T03:16:40,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742371_1547 (size=12151) 2024-11-25T03:16:40,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-25T03:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:16:40,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-25T03:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/73f15b71719f45038a856884ab5ada3d 2024-11-25T03:16:40,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/fb6b69c2772849cb8dcde88d60318c20 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fb6b69c2772849cb8dcde88d60318c20 2024-11-25T03:16:40,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fb6b69c2772849cb8dcde88d60318c20, entries=200, sequenceid=151, filesize=38.8 K 2024-11-25T03:16:40,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/8fdf59517bbe4cdd9ce679cb21da5241 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/8fdf59517bbe4cdd9ce679cb21da5241 2024-11-25T03:16:40,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-25T03:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:40,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/8fdf59517bbe4cdd9ce679cb21da5241, entries=150, sequenceid=151, filesize=11.9 K 2024-11-25T03:16:40,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/73f15b71719f45038a856884ab5ada3d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/73f15b71719f45038a856884ab5ada3d 2024-11-25T03:16:40,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/73f15b71719f45038a856884ab5ada3d, entries=150, sequenceid=151, filesize=11.9 K 2024-11-25T03:16:40,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 08a856257124e7cc799bf8e479f21703 in 1660ms, sequenceid=151, compaction requested=true 2024-11-25T03:16:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:40,822 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:40,822 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:40,823 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:40,823 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:40,823 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/B is initiating minor compaction (all files) 2024-11-25T03:16:40,823 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/A is initiating minor compaction (all files) 2024-11-25T03:16:40,823 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/A in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,823 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/B in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,823 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/dbde371f0cb6410d8413d0aa462bf1bd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b54664de9cef4babaeaad57f4fe4a2db, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fb6b69c2772849cb8dcde88d60318c20] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=99.6 K 2024-11-25T03:16:40,823 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0fe55677b719460ba4db407dd57d9b59, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/d7ae1e727d7c454283a0ecc0b3c57971, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/8fdf59517bbe4cdd9ce679cb21da5241] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.6 K 2024-11-25T03:16:40,823 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,823 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/dbde371f0cb6410d8413d0aa462bf1bd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b54664de9cef4babaeaad57f4fe4a2db, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fb6b69c2772849cb8dcde88d60318c20] 2024-11-25T03:16:40,824 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe55677b719460ba4db407dd57d9b59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504595350 2024-11-25T03:16:40,824 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbde371f0cb6410d8413d0aa462bf1bd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504595350 2024-11-25T03:16:40,824 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ae1e727d7c454283a0ecc0b3c57971, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732504596051 2024-11-25T03:16:40,824 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting b54664de9cef4babaeaad57f4fe4a2db, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732504596051 2024-11-25T03:16:40,824 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fdf59517bbe4cdd9ce679cb21da5241, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732504598019 2024-11-25T03:16:40,825 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6b69c2772849cb8dcde88d60318c20, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732504598018 2024-11-25T03:16:40,829 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:16:40,830 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:40,831 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/0ec96aea2f8d4f6a988998616d95853d is 50, key is test_row_0/B:col10/1732504598019/Put/seqid=0 2024-11-25T03:16:40,833 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125b4413fc4f47443b0bc91aa8a80c99717_08a856257124e7cc799bf8e479f21703 store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:40,835 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125b4413fc4f47443b0bc91aa8a80c99717_08a856257124e7cc799bf8e479f21703, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:40,835 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b4413fc4f47443b0bc91aa8a80c99717_08a856257124e7cc799bf8e479f21703 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:40,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742372_1548 (size=12459) 2024-11-25T03:16:40,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742373_1549 (size=4469) 2024-11-25T03:16:40,851 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#A#compaction#466 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:40,851 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/09065238d58e4e418f51d276df60e353 is 175, key is test_row_0/A:col10/1732504598019/Put/seqid=0 2024-11-25T03:16:40,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742374_1550 (size=31413) 2024-11-25T03:16:40,855 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/0ec96aea2f8d4f6a988998616d95853d as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0ec96aea2f8d4f6a988998616d95853d 2024-11-25T03:16:40,861 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/B of 08a856257124e7cc799bf8e479f21703 into 0ec96aea2f8d4f6a988998616d95853d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:40,861 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:40,861 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/B, priority=13, startTime=1732504600822; duration=0sec 2024-11-25T03:16:40,861 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/09065238d58e4e418f51d276df60e353 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/09065238d58e4e418f51d276df60e353 2024-11-25T03:16:40,861 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:40,861 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:B 2024-11-25T03:16:40,862 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:40,863 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:40,863 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/C is initiating minor compaction (all files) 2024-11-25T03:16:40,863 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/C in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,863 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/55932c66fc6e4a10a142d257dfd903b4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/f0cca49c2ba043119461dbdb8a4385c1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/73f15b71719f45038a856884ab5ada3d] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.6 K 2024-11-25T03:16:40,863 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 55932c66fc6e4a10a142d257dfd903b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732504595350 2024-11-25T03:16:40,864 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f0cca49c2ba043119461dbdb8a4385c1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732504596051 2024-11-25T03:16:40,864 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 73f15b71719f45038a856884ab5ada3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732504598019 2024-11-25T03:16:40,867 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/A of 08a856257124e7cc799bf8e479f21703 into 09065238d58e4e418f51d276df60e353(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:40,867 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:40,867 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/A, priority=13, startTime=1732504600822; duration=0sec 2024-11-25T03:16:40,867 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:40,868 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:A 2024-11-25T03:16:40,872 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:40,872 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8882201c01ac44e1ae08ddb4c2eb0dc7 is 50, key is test_row_0/C:col10/1732504598019/Put/seqid=0 2024-11-25T03:16:40,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742375_1551 (size=12459) 2024-11-25T03:16:40,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-25T03:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:40,968 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:40,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:40,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:40,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:40,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c9f54505417a4b388665351781ab2663_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504599185/Put/seqid=0 2024-11-25T03:16:40,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742376_1552 (size=12304) 2024-11-25T03:16:41,280 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8882201c01ac44e1ae08ddb4c2eb0dc7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8882201c01ac44e1ae08ddb4c2eb0dc7 2024-11-25T03:16:41,283 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/C of 08a856257124e7cc799bf8e479f21703 into 8882201c01ac44e1ae08ddb4c2eb0dc7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:41,284 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:41,284 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/C, priority=13, startTime=1732504600822; duration=0sec 2024-11-25T03:16:41,284 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:41,284 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:C 2024-11-25T03:16:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:16:41,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:41,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504661360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504661362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504661364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:41,383 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c9f54505417a4b388665351781ab2663_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c9f54505417a4b388665351781ab2663_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/747ea9ac2b6144808a164cbfaeeebac5, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/747ea9ac2b6144808a164cbfaeeebac5 is 175, key is test_row_0/A:col10/1732504599185/Put/seqid=0 2024-11-25T03:16:41,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742377_1553 (size=31105) 2024-11-25T03:16:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504661470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504661470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504661470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504661680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504661680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504661680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,788 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/747ea9ac2b6144808a164cbfaeeebac5 2024-11-25T03:16:41,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/9b65d7c8f470427eb65d901ce5a79d1b is 50, key is test_row_0/B:col10/1732504599185/Put/seqid=0 2024-11-25T03:16:41,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742378_1554 (size=12151) 2024-11-25T03:16:41,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504661990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504661990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504661990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,199 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/9b65d7c8f470427eb65d901ce5a79d1b 2024-11-25T03:16:42,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/1dcc28e282954cc2a8773dcbc8b478af is 50, key is test_row_0/C:col10/1732504599185/Put/seqid=0 2024-11-25T03:16:42,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742379_1555 (size=12151) 2024-11-25T03:16:42,211 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/1dcc28e282954cc2a8773dcbc8b478af 2024-11-25T03:16:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/747ea9ac2b6144808a164cbfaeeebac5 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/747ea9ac2b6144808a164cbfaeeebac5 2024-11-25T03:16:42,221 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/747ea9ac2b6144808a164cbfaeeebac5, entries=150, sequenceid=171, filesize=30.4 K 2024-11-25T03:16:42,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/9b65d7c8f470427eb65d901ce5a79d1b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/9b65d7c8f470427eb65d901ce5a79d1b 2024-11-25T03:16:42,225 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/9b65d7c8f470427eb65d901ce5a79d1b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-25T03:16:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/1dcc28e282954cc2a8773dcbc8b478af as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/1dcc28e282954cc2a8773dcbc8b478af 2024-11-25T03:16:42,229 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/1dcc28e282954cc2a8773dcbc8b478af, entries=150, sequenceid=171, filesize=11.9 K 2024-11-25T03:16:42,230 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 08a856257124e7cc799bf8e479f21703 in 1262ms, sequenceid=171, compaction requested=false 2024-11-25T03:16:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-25T03:16:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-25T03:16:42,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-25T03:16:42,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-25T03:16:42,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.0320 sec 2024-11-25T03:16:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:16:42,350 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:42,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-25T03:16:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T03:16:42,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:42,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:42,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T03:16:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:42,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:16:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:42,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b2a45c27c5bd41bd802d5da105d124ef_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504602501/Put/seqid=0 2024-11-25T03:16:42,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742380_1556 (size=14794) 2024-11-25T03:16:42,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:42,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b2a45c27c5bd41bd802d5da105d124ef_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b2a45c27c5bd41bd802d5da105d124ef_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:42,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b36403c75ae74872b8d1910ce6096ed0, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:42,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b36403c75ae74872b8d1910ce6096ed0 is 175, key is test_row_0/A:col10/1732504602501/Put/seqid=0 2024-11-25T03:16:42,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742381_1557 (size=39749) 2024-11-25T03:16:42,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504662524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504662528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504662530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504662640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504662640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504662641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T03:16:42,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:42,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:42,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504662850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504662850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504662850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:42,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b36403c75ae74872b8d1910ce6096ed0 2024-11-25T03:16:42,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/b9d5af3b3b2e4697a1413dfc1581b2c1 is 50, key is test_row_0/B:col10/1732504602501/Put/seqid=0 2024-11-25T03:16:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742382_1558 (size=12151) 2024-11-25T03:16:42,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:42,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T03:16:43,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504663169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504663170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504663170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:43,187 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:16:43,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/b9d5af3b3b2e4697a1413dfc1581b2c1 2024-11-25T03:16:43,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/d5e36729e18d408c9daca60e109c1b9b is 50, key is test_row_0/C:col10/1732504602501/Put/seqid=0 2024-11-25T03:16:43,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742383_1559 (size=12151) 2024-11-25T03:16:43,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T03:16:43,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504663680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504663680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504663680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:43,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:43,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/d5e36729e18d408c9daca60e109c1b9b 2024-11-25T03:16:43,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/b36403c75ae74872b8d1910ce6096ed0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b36403c75ae74872b8d1910ce6096ed0 2024-11-25T03:16:43,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b36403c75ae74872b8d1910ce6096ed0, entries=200, sequenceid=188, filesize=38.8 K 2024-11-25T03:16:43,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/b9d5af3b3b2e4697a1413dfc1581b2c1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/b9d5af3b3b2e4697a1413dfc1581b2c1 2024-11-25T03:16:43,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/b9d5af3b3b2e4697a1413dfc1581b2c1, entries=150, sequenceid=188, filesize=11.9 K 2024-11-25T03:16:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/d5e36729e18d408c9daca60e109c1b9b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d5e36729e18d408c9daca60e109c1b9b 2024-11-25T03:16:43,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d5e36729e18d408c9daca60e109c1b9b, entries=150, sequenceid=188, filesize=11.9 K 2024-11-25T03:16:43,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 08a856257124e7cc799bf8e479f21703 in 1263ms, sequenceid=188, compaction requested=true 2024-11-25T03:16:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:43,765 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:43,765 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:43,767 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:43,767 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/A is initiating minor compaction (all files) 2024-11-25T03:16:43,767 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:43,767 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/A in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,767 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/B is initiating minor compaction (all files) 2024-11-25T03:16:43,767 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/B in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,767 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/09065238d58e4e418f51d276df60e353, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/747ea9ac2b6144808a164cbfaeeebac5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b36403c75ae74872b8d1910ce6096ed0] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=99.9 K 2024-11-25T03:16:43,767 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0ec96aea2f8d4f6a988998616d95853d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/9b65d7c8f470427eb65d901ce5a79d1b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/b9d5af3b3b2e4697a1413dfc1581b2c1] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.9 K 2024-11-25T03:16:43,767 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,767 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/09065238d58e4e418f51d276df60e353, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/747ea9ac2b6144808a164cbfaeeebac5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b36403c75ae74872b8d1910ce6096ed0] 2024-11-25T03:16:43,768 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec96aea2f8d4f6a988998616d95853d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732504598019 2024-11-25T03:16:43,768 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09065238d58e4e418f51d276df60e353, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732504598019 2024-11-25T03:16:43,768 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b65d7c8f470427eb65d901ce5a79d1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732504599180 2024-11-25T03:16:43,768 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 747ea9ac2b6144808a164cbfaeeebac5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732504599180 2024-11-25T03:16:43,768 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36403c75ae74872b8d1910ce6096ed0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732504601358 2024-11-25T03:16:43,769 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d5af3b3b2e4697a1413dfc1581b2c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732504601360 2024-11-25T03:16:43,788 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:43,790 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125de0b4927a01542868874b9b71a828459_08a856257124e7cc799bf8e479f21703 store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:43,790 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#B#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:43,790 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/cd923c4ef35e4226ae4c6cb5d57061ca is 50, key is test_row_0/B:col10/1732504602501/Put/seqid=0 2024-11-25T03:16:43,791 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125de0b4927a01542868874b9b71a828459_08a856257124e7cc799bf8e479f21703, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:43,792 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de0b4927a01542868874b9b71a828459_08a856257124e7cc799bf8e479f21703 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:43,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742384_1560 (size=12561) 2024-11-25T03:16:43,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742385_1561 (size=4469) 2024-11-25T03:16:43,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-25T03:16:43,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:43,881 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T03:16:43,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:43,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:43,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:43,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:43,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:43,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aaf9832233bb4aacb2dbb8d971bbd335_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504602528/Put/seqid=0 2024-11-25T03:16:43,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742386_1562 (size=12304) 2024-11-25T03:16:44,205 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:44,206 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/cd923c4ef35e4226ae4c6cb5d57061ca as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cd923c4ef35e4226ae4c6cb5d57061ca 2024-11-25T03:16:44,206 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/3932510947764054a970c1f836a51e7f is 175, key is test_row_0/A:col10/1732504602501/Put/seqid=0 2024-11-25T03:16:44,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742387_1563 (size=31515) 2024-11-25T03:16:44,211 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/B of 08a856257124e7cc799bf8e479f21703 into cd923c4ef35e4226ae4c6cb5d57061ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:44,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:44,211 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/B, priority=13, startTime=1732504603765; duration=0sec 2024-11-25T03:16:44,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:44,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:B 2024-11-25T03:16:44,211 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:44,212 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:44,212 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/C is initiating minor compaction (all files) 2024-11-25T03:16:44,212 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/C in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:44,212 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8882201c01ac44e1ae08ddb4c2eb0dc7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/1dcc28e282954cc2a8773dcbc8b478af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d5e36729e18d408c9daca60e109c1b9b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=35.9 K 2024-11-25T03:16:44,213 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8882201c01ac44e1ae08ddb4c2eb0dc7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732504598019 2024-11-25T03:16:44,215 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dcc28e282954cc2a8773dcbc8b478af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732504599180 2024-11-25T03:16:44,218 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/3932510947764054a970c1f836a51e7f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3932510947764054a970c1f836a51e7f 2024-11-25T03:16:44,218 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting d5e36729e18d408c9daca60e109c1b9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732504601360 2024-11-25T03:16:44,222 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/A of 08a856257124e7cc799bf8e479f21703 into 3932510947764054a970c1f836a51e7f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:44,222 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:44,223 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/A, priority=13, startTime=1732504603765; duration=0sec 2024-11-25T03:16:44,223 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:44,223 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:A 2024-11-25T03:16:44,226 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:44,227 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/3a1070939c2c46c192829150ec7cec09 is 50, key is test_row_0/C:col10/1732504602501/Put/seqid=0 2024-11-25T03:16:44,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742388_1564 (size=12561) 2024-11-25T03:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:44,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:44,295 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aaf9832233bb4aacb2dbb8d971bbd335_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aaf9832233bb4aacb2dbb8d971bbd335_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:44,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/cbcdb0ab1a7f43b68129c3d3c52b22a9, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:44,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/cbcdb0ab1a7f43b68129c3d3c52b22a9 is 175, key is test_row_0/A:col10/1732504602528/Put/seqid=0 2024-11-25T03:16:44,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742389_1565 (size=31105) 2024-11-25T03:16:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504664338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:44,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504664449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T03:16:44,635 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/3a1070939c2c46c192829150ec7cec09 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/3a1070939c2c46c192829150ec7cec09 2024-11-25T03:16:44,639 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/C of 08a856257124e7cc799bf8e479f21703 into 3a1070939c2c46c192829150ec7cec09(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:44,639 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:44,639 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/C, priority=13, startTime=1732504603765; duration=0sec 2024-11-25T03:16:44,639 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:44,639 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:C 2024-11-25T03:16:44,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504664660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504664700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504664700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504664700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:44,703 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:44,704 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:44,704 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:44,710 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/cbcdb0ab1a7f43b68129c3d3c52b22a9 2024-11-25T03:16:44,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/4bfabcc15bdf4f0393b4f991aa3e4b85 is 50, key is test_row_0/B:col10/1732504602528/Put/seqid=0 2024-11-25T03:16:44,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742390_1566 (size=12151) 2024-11-25T03:16:44,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:44,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504664970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:45,122 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/4bfabcc15bdf4f0393b4f991aa3e4b85 2024-11-25T03:16:45,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/c884ecc264f248bbad0cc1d368de01c0 is 50, key is test_row_0/C:col10/1732504602528/Put/seqid=0 2024-11-25T03:16:45,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742391_1567 (size=12151) 2024-11-25T03:16:45,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504665480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:45,541 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/c884ecc264f248bbad0cc1d368de01c0 2024-11-25T03:16:45,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/cbcdb0ab1a7f43b68129c3d3c52b22a9 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/cbcdb0ab1a7f43b68129c3d3c52b22a9 2024-11-25T03:16:45,548 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/cbcdb0ab1a7f43b68129c3d3c52b22a9, entries=150, sequenceid=204, filesize=30.4 K 2024-11-25T03:16:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/4bfabcc15bdf4f0393b4f991aa3e4b85 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/4bfabcc15bdf4f0393b4f991aa3e4b85 2024-11-25T03:16:45,552 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/4bfabcc15bdf4f0393b4f991aa3e4b85, entries=150, sequenceid=204, filesize=11.9 K 2024-11-25T03:16:45,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/c884ecc264f248bbad0cc1d368de01c0 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/c884ecc264f248bbad0cc1d368de01c0 2024-11-25T03:16:45,556 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/c884ecc264f248bbad0cc1d368de01c0, entries=150, sequenceid=204, filesize=11.9 K 2024-11-25T03:16:45,557 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 08a856257124e7cc799bf8e479f21703 in 1676ms, sequenceid=204, compaction requested=false 2024-11-25T03:16:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-25T03:16:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-25T03:16:45,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-25T03:16:45,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2070 sec 2024-11-25T03:16:45,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.2110 sec 2024-11-25T03:16:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:46,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T03:16:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:46,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a9d603f10274497d9ea209f5230bbcfa_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504606491/Put/seqid=0 2024-11-25T03:16:46,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742392_1568 (size=14794) 2024-11-25T03:16:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T03:16:46,510 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:46,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-25T03:16:46,512 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:16:46,513 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:46,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:46,514 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44623 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=38a5d2077268,43357,1732504425108, table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:16:46,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504666549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:16:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504666660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:46,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-25T03:16:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:46,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504666710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:46,713 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:46,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504666730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:46,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504666730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:46,734 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:46,734 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:46,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-25T03:16:46,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:46,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:16:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504666870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:46,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:46,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a9d603f10274497d9ea209f5230bbcfa_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a9d603f10274497d9ea209f5230bbcfa_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:46,914 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/e01b29bf0810443b807e5020c1d36c03, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:46,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/e01b29bf0810443b807e5020c1d36c03 is 175, key is test_row_0/A:col10/1732504606491/Put/seqid=0 2024-11-25T03:16:46,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742393_1569 (size=39749) 2024-11-25T03:16:46,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/e01b29bf0810443b807e5020c1d36c03 2024-11-25T03:16:46,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/a2056110d10f489e90dbb56cb67f8483 is 50, key is test_row_0/B:col10/1732504606491/Put/seqid=0 2024-11-25T03:16:46,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-25T03:16:46,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:46,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:46,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:46,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:46,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742394_1570 (size=12151) 2024-11-25T03:16:47,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/a2056110d10f489e90dbb56cb67f8483 2024-11-25T03:16:47,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/9b93fcfc51db4c98b1c7e63debd43d7b is 50, key is test_row_0/C:col10/1732504606491/Put/seqid=0 2024-11-25T03:16:47,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742395_1571 (size=12151) 2024-11-25T03:16:47,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/9b93fcfc51db4c98b1c7e63debd43d7b 2024-11-25T03:16:47,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/e01b29bf0810443b807e5020c1d36c03 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/e01b29bf0810443b807e5020c1d36c03 2024-11-25T03:16:47,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/e01b29bf0810443b807e5020c1d36c03, entries=200, sequenceid=224, filesize=38.8 K 2024-11-25T03:16:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/a2056110d10f489e90dbb56cb67f8483 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a2056110d10f489e90dbb56cb67f8483 2024-11-25T03:16:47,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a2056110d10f489e90dbb56cb67f8483, entries=150, sequenceid=224, filesize=11.9 K 2024-11-25T03:16:47,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/9b93fcfc51db4c98b1c7e63debd43d7b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/9b93fcfc51db4c98b1c7e63debd43d7b 2024-11-25T03:16:47,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/9b93fcfc51db4c98b1c7e63debd43d7b, entries=150, sequenceid=224, filesize=11.9 K 2024-11-25T03:16:47,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 08a856257124e7cc799bf8e479f21703 in 566ms, sequenceid=224, compaction requested=true 2024-11-25T03:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:47,058 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:47,058 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:47,064 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:47,064 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/A is initiating minor compaction (all files) 2024-11-25T03:16:47,064 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/A in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:47,064 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3932510947764054a970c1f836a51e7f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/cbcdb0ab1a7f43b68129c3d3c52b22a9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/e01b29bf0810443b807e5020c1d36c03] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=100.0 K 2024-11-25T03:16:47,064 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:47,065 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3932510947764054a970c1f836a51e7f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/cbcdb0ab1a7f43b68129c3d3c52b22a9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/e01b29bf0810443b807e5020c1d36c03] 2024-11-25T03:16:47,065 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:47,065 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3932510947764054a970c1f836a51e7f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732504601360 2024-11-25T03:16:47,065 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/B is initiating minor compaction (all files) 2024-11-25T03:16:47,065 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/B in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:47,066 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cd923c4ef35e4226ae4c6cb5d57061ca, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/4bfabcc15bdf4f0393b4f991aa3e4b85, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a2056110d10f489e90dbb56cb67f8483] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=36.0 K 2024-11-25T03:16:47,066 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting cd923c4ef35e4226ae4c6cb5d57061ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732504601360 2024-11-25T03:16:47,066 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbcdb0ab1a7f43b68129c3d3c52b22a9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732504602522 2024-11-25T03:16:47,067 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfabcc15bdf4f0393b4f991aa3e4b85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732504602522 2024-11-25T03:16:47,067 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting e01b29bf0810443b807e5020c1d36c03, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732504604318 2024-11-25T03:16:47,067 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a2056110d10f489e90dbb56cb67f8483, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732504604331 2024-11-25T03:16:47,082 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:47,085 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/f98b07c328cd404db2faaf5d14027669 is 50, key is test_row_0/B:col10/1732504606491/Put/seqid=0 2024-11-25T03:16:47,102 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:47,120 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125e77b34d0f13b46749fd4239491f4e918_08a856257124e7cc799bf8e479f21703 store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:47,122 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125e77b34d0f13b46749fd4239491f4e918_08a856257124e7cc799bf8e479f21703, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:47,122 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e77b34d0f13b46749fd4239491f4e918_08a856257124e7cc799bf8e479f21703 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:47,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-25T03:16:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:47,128 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T03:16:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:16:47,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742396_1572 (size=12663) 2024-11-25T03:16:47,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:47,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742397_1573 (size=4469) 2024-11-25T03:16:47,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d1a73ed115ac4eceb89c06e5a3ab27b9_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504606542/Put/seqid=0 2024-11-25T03:16:47,213 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#A#compaction#484 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-25T03:16:47,214 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/d137fb4372cd4e65a155a61e430feb39 is 175, key is test_row_0/A:col10/1732504606491/Put/seqid=0 2024-11-25T03:16:47,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742398_1574 (size=12304) 2024-11-25T03:16:47,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742399_1575 (size=31617) 2024-11-25T03:16:47,240 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/d137fb4372cd4e65a155a61e430feb39 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/d137fb4372cd4e65a155a61e430feb39 2024-11-25T03:16:47,245 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/A of 08a856257124e7cc799bf8e479f21703 into d137fb4372cd4e65a155a61e430feb39(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:47,245 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:47,245 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/A, priority=13, startTime=1732504607058; duration=0sec 2024-11-25T03:16:47,245 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:47,245 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:A 2024-11-25T03:16:47,245 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:47,245 DEBUG [regionserver/38a5d2077268:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-25T03:16:47,246 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:47,246 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/C is initiating minor compaction (all files) 2024-11-25T03:16:47,246 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/C in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:47,246 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/3a1070939c2c46c192829150ec7cec09, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/c884ecc264f248bbad0cc1d368de01c0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/9b93fcfc51db4c98b1c7e63debd43d7b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=36.0 K 2024-11-25T03:16:47,246 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a1070939c2c46c192829150ec7cec09, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732504601360 2024-11-25T03:16:47,247 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting c884ecc264f248bbad0cc1d368de01c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732504602522 2024-11-25T03:16:47,247 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b93fcfc51db4c98b1c7e63debd43d7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732504604331 2024-11-25T03:16:47,252 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#C#compaction#486 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:16:47,253 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/01e529e30e2e4b66a181a9dd3398f613 is 50, key is test_row_0/C:col10/1732504606491/Put/seqid=0 2024-11-25T03:16:47,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742400_1576 (size=12663) 2024-11-25T03:16:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504667271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504667380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:47,562 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/f98b07c328cd404db2faaf5d14027669 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f98b07c328cd404db2faaf5d14027669 2024-11-25T03:16:47,566 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/B of 08a856257124e7cc799bf8e479f21703 into f98b07c328cd404db2faaf5d14027669(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:47,566 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:47,566 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/B, priority=13, startTime=1732504607058; duration=0sec 2024-11-25T03:16:47,566 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:47,566 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:B 2024-11-25T03:16:47,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504667590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:47,629 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d1a73ed115ac4eceb89c06e5a3ab27b9_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d1a73ed115ac4eceb89c06e5a3ab27b9_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/efef3a32c8d44b4a89e98f6d7ff7f483, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/efef3a32c8d44b4a89e98f6d7ff7f483 is 175, key is test_row_0/A:col10/1732504606542/Put/seqid=0 2024-11-25T03:16:47,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742401_1577 (size=31105) 2024-11-25T03:16:47,634 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/efef3a32c8d44b4a89e98f6d7ff7f483 2024-11-25T03:16:47,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/91fdf62af25b4301a9096619e29cf1fc is 50, key is test_row_0/B:col10/1732504606542/Put/seqid=0 2024-11-25T03:16:47,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742402_1578 (size=12151) 2024-11-25T03:16:47,645 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/91fdf62af25b4301a9096619e29cf1fc 2024-11-25T03:16:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:16:47,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/579d495118554ab69cf69df2823331c8 is 50, key is test_row_0/C:col10/1732504606542/Put/seqid=0 2024-11-25T03:16:47,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742403_1579 (size=12151) 2024-11-25T03:16:47,661 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/01e529e30e2e4b66a181a9dd3398f613 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/01e529e30e2e4b66a181a9dd3398f613 2024-11-25T03:16:47,665 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/C of 08a856257124e7cc799bf8e479f21703 into 01e529e30e2e4b66a181a9dd3398f613(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:47,665 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:47,665 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/C, priority=13, startTime=1732504607058; duration=0sec 2024-11-25T03:16:47,665 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:47,665 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:C 2024-11-25T03:16:47,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504667900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:48,057 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/579d495118554ab69cf69df2823331c8 2024-11-25T03:16:48,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/efef3a32c8d44b4a89e98f6d7ff7f483 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/efef3a32c8d44b4a89e98f6d7ff7f483 2024-11-25T03:16:48,069 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/efef3a32c8d44b4a89e98f6d7ff7f483, entries=150, sequenceid=239, filesize=30.4 K 2024-11-25T03:16:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/91fdf62af25b4301a9096619e29cf1fc as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/91fdf62af25b4301a9096619e29cf1fc 2024-11-25T03:16:48,075 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/91fdf62af25b4301a9096619e29cf1fc, entries=150, sequenceid=239, filesize=11.9 K 2024-11-25T03:16:48,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/579d495118554ab69cf69df2823331c8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/579d495118554ab69cf69df2823331c8 2024-11-25T03:16:48,082 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/579d495118554ab69cf69df2823331c8, entries=150, sequenceid=239, filesize=11.9 K 2024-11-25T03:16:48,083 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 08a856257124e7cc799bf8e479f21703 in 955ms, sequenceid=239, compaction requested=false 2024-11-25T03:16:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-25T03:16:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-25T03:16:48,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-25T03:16:48,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-11-25T03:16:48,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.5770 sec 2024-11-25T03:16:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:48,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T03:16:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:48,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125912e0c2dc1a648e6a38d38dd92638de5_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504607269/Put/seqid=0 2024-11-25T03:16:48,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742404_1580 (size=14844) 2024-11-25T03:16:48,237 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:48,240 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125912e0c2dc1a648e6a38d38dd92638de5_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125912e0c2dc1a648e6a38d38dd92638de5_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:48,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:48,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b is 175, key is test_row_0/A:col10/1732504607269/Put/seqid=0 2024-11-25T03:16:48,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504668254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:48,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742405_1581 (size=39799) 2024-11-25T03:16:48,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504668370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504668410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504668580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:16:48,670 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:48,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-25T03:16:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-25T03:16:48,672 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:16:48,673 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:16:48,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:16:48,682 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b 2024-11-25T03:16:48,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/462c7ae2b83e4d569e50499e0c60f75a is 50, key is test_row_0/B:col10/1732504607269/Put/seqid=0 2024-11-25T03:16:48,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742406_1582 (size=12201) 2024-11-25T03:16:48,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/462c7ae2b83e4d569e50499e0c60f75a 2024-11-25T03:16:48,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/a3f52af6f4844ccd8754ab4ce9684f39 is 50, key is test_row_0/C:col10/1732504607269/Put/seqid=0 2024-11-25T03:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-25T03:16:48,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742407_1583 (size=12201) 2024-11-25T03:16:48,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-25T03:16:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:48,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:48,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504668890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:48,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-25T03:16:48,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:48,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:48,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:48,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-25T03:16:49,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-25T03:16:49,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:49,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:49,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:49,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:16:49,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/a3f52af6f4844ccd8754ab4ce9684f39 2024-11-25T03:16:49,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b 2024-11-25T03:16:49,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b, entries=200, sequenceid=260, filesize=38.9 K 2024-11-25T03:16:49,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/462c7ae2b83e4d569e50499e0c60f75a as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/462c7ae2b83e4d569e50499e0c60f75a 2024-11-25T03:16:49,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/462c7ae2b83e4d569e50499e0c60f75a, entries=150, sequenceid=260, filesize=11.9 K 2024-11-25T03:16:49,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/a3f52af6f4844ccd8754ab4ce9684f39 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/a3f52af6f4844ccd8754ab4ce9684f39 2024-11-25T03:16:49,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/a3f52af6f4844ccd8754ab4ce9684f39, entries=150, sequenceid=260, filesize=11.9 K 2024-11-25T03:16:49,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 08a856257124e7cc799bf8e479f21703 in 1067ms, sequenceid=260, compaction requested=true 2024-11-25T03:16:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:49,229 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:49,229 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:49,230 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:49,230 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:49,230 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/B is initiating minor compaction (all files) 2024-11-25T03:16:49,230 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/A is initiating minor compaction (all files) 2024-11-25T03:16:49,230 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/B in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:49,230 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/A in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:49,230 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f98b07c328cd404db2faaf5d14027669, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/91fdf62af25b4301a9096619e29cf1fc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/462c7ae2b83e4d569e50499e0c60f75a] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=36.1 K 2024-11-25T03:16:49,230 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/d137fb4372cd4e65a155a61e430feb39, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/efef3a32c8d44b4a89e98f6d7ff7f483, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=100.1 K 2024-11-25T03:16:49,230 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:49,230 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/d137fb4372cd4e65a155a61e430feb39, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/efef3a32c8d44b4a89e98f6d7ff7f483, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b] 2024-11-25T03:16:49,230 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f98b07c328cd404db2faaf5d14027669, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732504604331 2024-11-25T03:16:49,230 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting d137fb4372cd4e65a155a61e430feb39, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732504604331 2024-11-25T03:16:49,231 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 91fdf62af25b4301a9096619e29cf1fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732504606516 2024-11-25T03:16:49,231 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting efef3a32c8d44b4a89e98f6d7ff7f483, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732504606516 2024-11-25T03:16:49,231 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 462c7ae2b83e4d569e50499e0c60f75a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504607246 2024-11-25T03:16:49,231 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ddf1a18ddd542a4a2afd88c6dd0bd2b, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504607246 2024-11-25T03:16:49,237 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:49,239 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:16:49,240 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/96267559dd1d4e5a83a7b722df3e7d33 is 50, key is test_row_0/B:col10/1732504607269/Put/seqid=0 2024-11-25T03:16:49,241 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125f1eb0634b9ef4c42b3658f520167cad7_08a856257124e7cc799bf8e479f21703 store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:49,243 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125f1eb0634b9ef4c42b3658f520167cad7_08a856257124e7cc799bf8e479f21703, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:49,243 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f1eb0634b9ef4c42b3658f520167cad7_08a856257124e7cc799bf8e479f21703 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:49,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742408_1584 (size=12815) 2024-11-25T03:16:49,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742409_1585 (size=4469) 2024-11-25T03:16:49,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-25T03:16:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:49,285 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T03:16:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aa590b1c84224a1f9cc85d5f9154a233_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504608251/Put/seqid=0 2024-11-25T03:16:49,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742410_1586 (size=12454) 2024-11-25T03:16:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-25T03:16:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:49,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. as already flushing 2024-11-25T03:16:49,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504669468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:49,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504669469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:49,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504669580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504669580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:49,649 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:16:49,649 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/ee0135c9107e45b8bec33024cd83c4a2 is 175, key is test_row_0/A:col10/1732504607269/Put/seqid=0 2024-11-25T03:16:49,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742411_1587 (size=31769) 2024-11-25T03:16:49,655 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/96267559dd1d4e5a83a7b722df3e7d33 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/96267559dd1d4e5a83a7b722df3e7d33 2024-11-25T03:16:49,659 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/B of 08a856257124e7cc799bf8e479f21703 into 96267559dd1d4e5a83a7b722df3e7d33(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:49,659 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:49,659 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/B, priority=13, startTime=1732504609229; duration=0sec 2024-11-25T03:16:49,659 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:49,659 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:B 2024-11-25T03:16:49,660 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:49,660 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:49,660 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/C is initiating minor compaction (all files) 2024-11-25T03:16:49,660 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/C in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:49,661 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/01e529e30e2e4b66a181a9dd3398f613, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/579d495118554ab69cf69df2823331c8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/a3f52af6f4844ccd8754ab4ce9684f39] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=36.1 K 2024-11-25T03:16:49,661 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e529e30e2e4b66a181a9dd3398f613, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732504604331 2024-11-25T03:16:49,661 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 579d495118554ab69cf69df2823331c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732504606516 2024-11-25T03:16:49,662 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f52af6f4844ccd8754ab4ce9684f39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504607246 2024-11-25T03:16:49,667 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:16:49,667 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/fd89a77134bf460d802ee439c916d889 is 50, key is test_row_0/C:col10/1732504607269/Put/seqid=0 2024-11-25T03:16:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742412_1588 (size=12815) 2024-11-25T03:16:49,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:49,699 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aa590b1c84224a1f9cc85d5f9154a233_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aa590b1c84224a1f9cc85d5f9154a233_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:49,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/7e492615cdf94d908a8e35ba5359fce7, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/7e492615cdf94d908a8e35ba5359fce7 is 175, key is test_row_0/A:col10/1732504608251/Put/seqid=0 2024-11-25T03:16:49,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742413_1589 (size=31255) 2024-11-25T03:16:49,716 INFO [Thread-2317 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2317. 2024-11-25T03:16:49,716 DEBUG [Thread-2317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:49,716 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,716 INFO [Thread-2313 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2313. 2024-11-25T03:16:49,716 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,716 DEBUG [Thread-2313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:49,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:49,716 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,716 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:49,718 INFO [Thread-2311 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2311. 2024-11-25T03:16:49,718 DEBUG [Thread-2311 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:49,718 DEBUG [Thread-2311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,718 DEBUG [Thread-2311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:49,720 INFO [Thread-2319 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2319. 2024-11-25T03:16:49,720 DEBUG [Thread-2319 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:49,720 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,720 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:49,720 INFO [Thread-2315 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2315. 2024-11-25T03:16:49,720 DEBUG [Thread-2315 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:49,720 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,720 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:49,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:49,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:49,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504669790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504669790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-25T03:16:50,056 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/ee0135c9107e45b8bec33024cd83c4a2 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ee0135c9107e45b8bec33024cd83c4a2 2024-11-25T03:16:50,059 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/A of 08a856257124e7cc799bf8e479f21703 into ee0135c9107e45b8bec33024cd83c4a2(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:50,059 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:50,059 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/A, priority=13, startTime=1732504609229; duration=0sec 2024-11-25T03:16:50,059 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:50,059 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:A 2024-11-25T03:16:50,073 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/fd89a77134bf460d802ee439c916d889 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/fd89a77134bf460d802ee439c916d889 2024-11-25T03:16:50,076 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/C of 08a856257124e7cc799bf8e479f21703 into fd89a77134bf460d802ee439c916d889(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:50,076 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:50,076 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/C, priority=13, startTime=1732504609229; duration=0sec 2024-11-25T03:16:50,076 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:50,076 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:C 2024-11-25T03:16:50,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:50,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504670100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504670100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:50,104 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/7e492615cdf94d908a8e35ba5359fce7 2024-11-25T03:16:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/f5a0340ababe49658881a8b42ee458b9 is 50, key is test_row_0/B:col10/1732504608251/Put/seqid=0 2024-11-25T03:16:50,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742414_1590 (size=12301) 2024-11-25T03:16:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/f5a0340ababe49658881a8b42ee458b9 2024-11-25T03:16:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/976ddf7657a448d484df4f10ef20c991 is 50, key is test_row_0/C:col10/1732504608251/Put/seqid=0 2024-11-25T03:16:50,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742415_1591 (size=12301) 2024-11-25T03:16:50,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:50,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1732504670610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52316 deadline: 1732504670610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1732504670730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:50,730 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:50,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1732504670760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:50,761 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_2' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:16:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1732504670770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 2024-11-25T03:16:50,771 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,43357,1732504425108 for 'test_row_1' in 08a856257124e7cc799bf8e479f21703 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:16:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 , the old value is region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:16:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., hostname=38a5d2077268,43357,1732504425108, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:16:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-25T03:16:50,920 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/976ddf7657a448d484df4f10ef20c991 2024-11-25T03:16:50,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/7e492615cdf94d908a8e35ba5359fce7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/7e492615cdf94d908a8e35ba5359fce7 2024-11-25T03:16:50,925 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/7e492615cdf94d908a8e35ba5359fce7, entries=150, sequenceid=274, filesize=30.5 K 2024-11-25T03:16:50,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/f5a0340ababe49658881a8b42ee458b9 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f5a0340ababe49658881a8b42ee458b9 2024-11-25T03:16:50,928 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f5a0340ababe49658881a8b42ee458b9, entries=150, sequenceid=274, filesize=12.0 K 2024-11-25T03:16:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/976ddf7657a448d484df4f10ef20c991 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/976ddf7657a448d484df4f10ef20c991 2024-11-25T03:16:50,931 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/976ddf7657a448d484df4f10ef20c991, entries=150, sequenceid=274, filesize=12.0 K 2024-11-25T03:16:50,931 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 08a856257124e7cc799bf8e479f21703 in 1646ms, sequenceid=274, compaction requested=false 2024-11-25T03:16:50,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:50,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:50,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-25T03:16:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-25T03:16:50,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-25T03:16:50,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2600 sec 2024-11-25T03:16:50,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.2650 sec 2024-11-25T03:16:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43357 {}] regionserver.HRegion(8855): Flush requested on 08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:51,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T03:16:51,621 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-11-25T03:16:51,621 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:51,621 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-25T03:16:51,621 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:51,621 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:16:51,621 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:51,621 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:51,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:16:51,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:51,622 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:16:51,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:51,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:16:51,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:51,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:16:51,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:16:51,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:16:51,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253145b60296f247dd9d5da0afeef20fea_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_0/A:col10/1732504611620/Put/seqid=0 2024-11-25T03:16:51,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742416_1592 (size=12454) 2024-11-25T03:16:52,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:16:52,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253145b60296f247dd9d5da0afeef20fea_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253145b60296f247dd9d5da0afeef20fea_08a856257124e7cc799bf8e479f21703 2024-11-25T03:16:52,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/ba2175f1b40e4e0f80ea374caef8ef86, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:52,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/ba2175f1b40e4e0f80ea374caef8ef86 is 175, key is test_row_0/A:col10/1732504611620/Put/seqid=0 2024-11-25T03:16:52,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742417_1593 (size=31255) 2024-11-25T03:16:52,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/ba2175f1b40e4e0f80ea374caef8ef86 2024-11-25T03:16:52,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/0310c7606f0d468a951281cc0a255a37 is 50, key is test_row_0/B:col10/1732504611620/Put/seqid=0 2024-11-25T03:16:52,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742418_1594 (size=12301) 2024-11-25T03:16:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-25T03:16:52,840 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:16:52,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/0310c7606f0d468a951281cc0a255a37 2024-11-25T03:16:52,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8108c04e038b400d98d2046c3e55a712 is 50, key is test_row_0/C:col10/1732504611620/Put/seqid=0 2024-11-25T03:16:52,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742419_1595 (size=12301) 2024-11-25T03:16:53,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8108c04e038b400d98d2046c3e55a712 2024-11-25T03:16:53,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/ba2175f1b40e4e0f80ea374caef8ef86 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ba2175f1b40e4e0f80ea374caef8ef86 2024-11-25T03:16:53,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ba2175f1b40e4e0f80ea374caef8ef86, entries=150, sequenceid=297, filesize=30.5 K 2024-11-25T03:16:53,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/0310c7606f0d468a951281cc0a255a37 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0310c7606f0d468a951281cc0a255a37 2024-11-25T03:16:53,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0310c7606f0d468a951281cc0a255a37, entries=150, sequenceid=297, filesize=12.0 K 2024-11-25T03:16:53,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/8108c04e038b400d98d2046c3e55a712 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8108c04e038b400d98d2046c3e55a712 2024-11-25T03:16:53,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8108c04e038b400d98d2046c3e55a712, entries=150, sequenceid=297, filesize=12.0 K 2024-11-25T03:16:53,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 08a856257124e7cc799bf8e479f21703 in 1645ms, sequenceid=297, compaction requested=true 2024-11-25T03:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:16:53,266 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08a856257124e7cc799bf8e479f21703:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:53,267 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:53,267 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:53,267 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/A is initiating minor compaction (all files) 2024-11-25T03:16:53,267 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:53,267 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/A in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:53,267 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/B is initiating minor compaction (all files) 2024-11-25T03:16:53,267 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/B in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:53,267 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ee0135c9107e45b8bec33024cd83c4a2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/7e492615cdf94d908a8e35ba5359fce7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ba2175f1b40e4e0f80ea374caef8ef86] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=92.1 K 2024-11-25T03:16:53,267 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:53,267 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/96267559dd1d4e5a83a7b722df3e7d33, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f5a0340ababe49658881a8b42ee458b9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0310c7606f0d468a951281cc0a255a37] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=36.5 K 2024-11-25T03:16:53,267 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. files: [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ee0135c9107e45b8bec33024cd83c4a2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/7e492615cdf94d908a8e35ba5359fce7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ba2175f1b40e4e0f80ea374caef8ef86] 2024-11-25T03:16:53,268 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee0135c9107e45b8bec33024cd83c4a2, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504607246 2024-11-25T03:16:53,268 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 96267559dd1d4e5a83a7b722df3e7d33, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504607246 2024-11-25T03:16:53,268 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e492615cdf94d908a8e35ba5359fce7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504608195 2024-11-25T03:16:53,268 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a0340ababe49658881a8b42ee458b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504608195 2024-11-25T03:16:53,268 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba2175f1b40e4e0f80ea374caef8ef86, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732504609447 2024-11-25T03:16:53,268 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 0310c7606f0d468a951281cc0a255a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732504609447 2024-11-25T03:16:53,277 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:16:53,277 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/6d8d9dcd722b44e5beaf53c6d0cfd53b is 50, key is test_row_0/B:col10/1732504611620/Put/seqid=0 2024-11-25T03:16:53,278 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:53,281 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411256bc99ce668d9434294023a54ab86f5c1_08a856257124e7cc799bf8e479f21703 store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:53,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742420_1596 (size=13017) 2024-11-25T03:16:53,285 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/6d8d9dcd722b44e5beaf53c6d0cfd53b as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/6d8d9dcd722b44e5beaf53c6d0cfd53b 2024-11-25T03:16:53,288 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/B of 08a856257124e7cc799bf8e479f21703 into 6d8d9dcd722b44e5beaf53c6d0cfd53b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:53,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:53,288 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/B, priority=13, startTime=1732504613266; duration=0sec 2024-11-25T03:16:53,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:16:53,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:B 2024-11-25T03:16:53,288 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:16:53,289 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:16:53,289 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1541): 08a856257124e7cc799bf8e479f21703/C is initiating minor compaction (all files) 2024-11-25T03:16:53,289 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08a856257124e7cc799bf8e479f21703/C in TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:16:53,289 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/fd89a77134bf460d802ee439c916d889, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/976ddf7657a448d484df4f10ef20c991, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8108c04e038b400d98d2046c3e55a712] into tmpdir=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp, totalSize=36.5 K 2024-11-25T03:16:53,289 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting fd89a77134bf460d802ee439c916d889, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732504607246 2024-11-25T03:16:53,290 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 976ddf7657a448d484df4f10ef20c991, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732504608195 2024-11-25T03:16:53,290 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] compactions.Compactor(225): Compacting 8108c04e038b400d98d2046c3e55a712, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732504609447 2024-11-25T03:16:53,299 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#C#compaction#503 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:16:53,300 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/52ed38e0f25d4d7ebddca70afc04ecb7 is 50, key is test_row_0/C:col10/1732504611620/Put/seqid=0 2024-11-25T03:16:53,302 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411256bc99ce668d9434294023a54ab86f5c1_08a856257124e7cc799bf8e479f21703, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:53,302 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256bc99ce668d9434294023a54ab86f5c1_08a856257124e7cc799bf8e479f21703 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:16:53,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742421_1597 (size=13017) 2024-11-25T03:16:53,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742422_1598 (size=4469) 2024-11-25T03:16:53,707 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/52ed38e0f25d4d7ebddca70afc04ecb7 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/52ed38e0f25d4d7ebddca70afc04ecb7 2024-11-25T03:16:53,709 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08a856257124e7cc799bf8e479f21703#A#compaction#502 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:16:53,710 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/8bd260fc1ce84dfb8b671be07f748fda is 175, key is test_row_0/A:col10/1732504611620/Put/seqid=0 2024-11-25T03:16:53,711 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/C of 08a856257124e7cc799bf8e479f21703 into 52ed38e0f25d4d7ebddca70afc04ecb7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:53,711 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:53,711 INFO [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/C, priority=13, startTime=1732504613266; duration=0sec 2024-11-25T03:16:53,711 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:53,711 DEBUG [RS:0;38a5d2077268:43357-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:C 2024-11-25T03:16:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742423_1599 (size=31971) 2024-11-25T03:16:54,117 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/8bd260fc1ce84dfb8b671be07f748fda as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8bd260fc1ce84dfb8b671be07f748fda 2024-11-25T03:16:54,120 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08a856257124e7cc799bf8e479f21703/A of 08a856257124e7cc799bf8e479f21703 into 8bd260fc1ce84dfb8b671be07f748fda(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:16:54,120 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08a856257124e7cc799bf8e479f21703: 2024-11-25T03:16:54,120 INFO [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., storeName=08a856257124e7cc799bf8e479f21703/A, priority=13, startTime=1732504613266; duration=0sec 2024-11-25T03:16:54,120 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:16:54,120 DEBUG [RS:0;38a5d2077268:43357-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08a856257124e7cc799bf8e479f21703:A 2024-11-25T03:17:00,761 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-25T03:17:00,761 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:17:00,761 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,761 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:17:00,781 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-25T03:17:00,781 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:17:00,781 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,781 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:17:00,861 INFO [Thread-2300 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2300. 2024-11-25T03:17:00,861 DEBUG [Thread-2300 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:17:00,861 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,861 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3266 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3175 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3178 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3266 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3188 2024-11-25T03:17:00,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:17:00,862 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:17:00,862 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:17:00,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:00,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:17:00,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:17:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:17:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-25T03:17:00,865 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504620865"}]},"ts":"1732504620865"} 2024-11-25T03:17:00,866 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:17:00,866 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:17:00,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:17:00,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, UNASSIGN}] 2024-11-25T03:17:00,868 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, UNASSIGN 2024-11-25T03:17:00,869 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=CLOSING, regionLocation=38a5d2077268,43357,1732504425108 2024-11-25T03:17:00,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, UNASSIGN because future has completed 2024-11-25T03:17:00,870 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:17:00,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108}] 2024-11-25T03:17:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-25T03:17:01,021 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:01,021 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 08a856257124e7cc799bf8e479f21703, disabling compactions & flushes 2024-11-25T03:17:01,022 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. after waiting 0 ms 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:17:01,022 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 08a856257124e7cc799bf8e479f21703 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=A 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=B 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 08a856257124e7cc799bf8e479f21703, store=C 2024-11-25T03:17:01,022 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T03:17:01,027 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112501924ece69e64042a2581d30d6284445_08a856257124e7cc799bf8e479f21703 is 50, key is test_row_1/A:col10/1732504620860/Put/seqid=0 2024-11-25T03:17:01,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742424_1600 (size=9914) 2024-11-25T03:17:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-25T03:17:01,430 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:17:01,433 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112501924ece69e64042a2581d30d6284445_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112501924ece69e64042a2581d30d6284445_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:01,434 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/901d18d29633482b85dadd7b9d616277, store: [table=TestAcidGuarantees family=A region=08a856257124e7cc799bf8e479f21703] 2024-11-25T03:17:01,435 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/901d18d29633482b85dadd7b9d616277 is 175, key is test_row_1/A:col10/1732504620860/Put/seqid=0 2024-11-25T03:17:01,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742425_1601 (size=22561) 2024-11-25T03:17:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-25T03:17:01,842 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/901d18d29633482b85dadd7b9d616277 2024-11-25T03:17:01,847 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/7e70a4d73680448dbec91904deee9b4f is 50, key is test_row_1/B:col10/1732504620860/Put/seqid=0 2024-11-25T03:17:01,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742426_1602 (size=9857) 2024-11-25T03:17:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-25T03:17:02,251 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/7e70a4d73680448dbec91904deee9b4f 2024-11-25T03:17:02,256 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/900adac488e3464bbeee54d7024a52fd is 50, key is test_row_1/C:col10/1732504620860/Put/seqid=0 2024-11-25T03:17:02,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742427_1603 (size=9857) 2024-11-25T03:17:02,659 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/900adac488e3464bbeee54d7024a52fd 2024-11-25T03:17:02,663 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/A/901d18d29633482b85dadd7b9d616277 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/901d18d29633482b85dadd7b9d616277 2024-11-25T03:17:02,665 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/901d18d29633482b85dadd7b9d616277, entries=100, sequenceid=306, filesize=22.0 K 2024-11-25T03:17:02,666 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/B/7e70a4d73680448dbec91904deee9b4f as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/7e70a4d73680448dbec91904deee9b4f 2024-11-25T03:17:02,668 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/7e70a4d73680448dbec91904deee9b4f, entries=100, sequenceid=306, filesize=9.6 K 2024-11-25T03:17:02,669 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/.tmp/C/900adac488e3464bbeee54d7024a52fd as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/900adac488e3464bbeee54d7024a52fd 2024-11-25T03:17:02,671 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/900adac488e3464bbeee54d7024a52fd, entries=100, sequenceid=306, filesize=9.6 K 2024-11-25T03:17:02,671 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 08a856257124e7cc799bf8e479f21703 in 1649ms, sequenceid=306, compaction requested=false 2024-11-25T03:17:02,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3add0679b0784ba0994dd5b3ab2cdcb2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/6f2e071549bf47ce91fd1919d9a04a99, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/94b2f65d0ae848208944e144df694da1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/a332729d2c364ecf85fa053a49f1d90f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fcc2f0a3461249bea0bb4fa361134c23, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/91d85bdef0c04ba2ab2e834bcbe0f91a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/dbde371f0cb6410d8413d0aa462bf1bd, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b54664de9cef4babaeaad57f4fe4a2db, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fb6b69c2772849cb8dcde88d60318c20, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/09065238d58e4e418f51d276df60e353, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/747ea9ac2b6144808a164cbfaeeebac5, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b36403c75ae74872b8d1910ce6096ed0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3932510947764054a970c1f836a51e7f, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/cbcdb0ab1a7f43b68129c3d3c52b22a9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/e01b29bf0810443b807e5020c1d36c03, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/d137fb4372cd4e65a155a61e430feb39, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/efef3a32c8d44b4a89e98f6d7ff7f483, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ee0135c9107e45b8bec33024cd83c4a2, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/7e492615cdf94d908a8e35ba5359fce7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ba2175f1b40e4e0f80ea374caef8ef86] to archive 2024-11-25T03:17:02,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:17:02,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3add0679b0784ba0994dd5b3ab2cdcb2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3add0679b0784ba0994dd5b3ab2cdcb2 2024-11-25T03:17:02,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/6f2e071549bf47ce91fd1919d9a04a99 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/6f2e071549bf47ce91fd1919d9a04a99 2024-11-25T03:17:02,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/94b2f65d0ae848208944e144df694da1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/94b2f65d0ae848208944e144df694da1 2024-11-25T03:17:02,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/a332729d2c364ecf85fa053a49f1d90f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/a332729d2c364ecf85fa053a49f1d90f 2024-11-25T03:17:02,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fcc2f0a3461249bea0bb4fa361134c23 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fcc2f0a3461249bea0bb4fa361134c23 2024-11-25T03:17:02,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/91d85bdef0c04ba2ab2e834bcbe0f91a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/91d85bdef0c04ba2ab2e834bcbe0f91a 2024-11-25T03:17:02,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/dbde371f0cb6410d8413d0aa462bf1bd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/dbde371f0cb6410d8413d0aa462bf1bd 2024-11-25T03:17:02,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b54664de9cef4babaeaad57f4fe4a2db to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b54664de9cef4babaeaad57f4fe4a2db 2024-11-25T03:17:02,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fb6b69c2772849cb8dcde88d60318c20 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/fb6b69c2772849cb8dcde88d60318c20 2024-11-25T03:17:02,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/09065238d58e4e418f51d276df60e353 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/09065238d58e4e418f51d276df60e353 2024-11-25T03:17:02,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/747ea9ac2b6144808a164cbfaeeebac5 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/747ea9ac2b6144808a164cbfaeeebac5 2024-11-25T03:17:02,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b36403c75ae74872b8d1910ce6096ed0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/b36403c75ae74872b8d1910ce6096ed0 2024-11-25T03:17:02,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3932510947764054a970c1f836a51e7f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/3932510947764054a970c1f836a51e7f 2024-11-25T03:17:02,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/cbcdb0ab1a7f43b68129c3d3c52b22a9 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/cbcdb0ab1a7f43b68129c3d3c52b22a9 2024-11-25T03:17:02,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/e01b29bf0810443b807e5020c1d36c03 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/e01b29bf0810443b807e5020c1d36c03 2024-11-25T03:17:02,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/d137fb4372cd4e65a155a61e430feb39 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/d137fb4372cd4e65a155a61e430feb39 2024-11-25T03:17:02,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/efef3a32c8d44b4a89e98f6d7ff7f483 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/efef3a32c8d44b4a89e98f6d7ff7f483 2024-11-25T03:17:02,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8ddf1a18ddd542a4a2afd88c6dd0bd2b 2024-11-25T03:17:02,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ee0135c9107e45b8bec33024cd83c4a2 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ee0135c9107e45b8bec33024cd83c4a2 2024-11-25T03:17:02,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/7e492615cdf94d908a8e35ba5359fce7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/7e492615cdf94d908a8e35ba5359fce7 2024-11-25T03:17:02,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ba2175f1b40e4e0f80ea374caef8ef86 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/ba2175f1b40e4e0f80ea374caef8ef86 2024-11-25T03:17:02,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cfad0a23620a48979d988da12e54f558, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a21bea1770fc45a983bd2e8adb7f4bfe, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/169dfc6cab2f4307a3b884a865bb3b7d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/18e660c3b898475da74683207416eb2a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0fe55677b719460ba4db407dd57d9b59, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/e281a111bb0f4391b795e6e32f6f92fa, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/d7ae1e727d7c454283a0ecc0b3c57971, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0ec96aea2f8d4f6a988998616d95853d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/8fdf59517bbe4cdd9ce679cb21da5241, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/9b65d7c8f470427eb65d901ce5a79d1b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cd923c4ef35e4226ae4c6cb5d57061ca, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/b9d5af3b3b2e4697a1413dfc1581b2c1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/4bfabcc15bdf4f0393b4f991aa3e4b85, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f98b07c328cd404db2faaf5d14027669, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a2056110d10f489e90dbb56cb67f8483, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/91fdf62af25b4301a9096619e29cf1fc, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/96267559dd1d4e5a83a7b722df3e7d33, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/462c7ae2b83e4d569e50499e0c60f75a, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f5a0340ababe49658881a8b42ee458b9, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0310c7606f0d468a951281cc0a255a37] to archive 2024-11-25T03:17:02,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:17:02,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/5b73e1dac6bd4dfd9c2a2035cb9f4bf3 2024-11-25T03:17:02,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cfad0a23620a48979d988da12e54f558 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cfad0a23620a48979d988da12e54f558 2024-11-25T03:17:02,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a21bea1770fc45a983bd2e8adb7f4bfe to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a21bea1770fc45a983bd2e8adb7f4bfe 2024-11-25T03:17:02,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/169dfc6cab2f4307a3b884a865bb3b7d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/169dfc6cab2f4307a3b884a865bb3b7d 2024-11-25T03:17:02,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/18e660c3b898475da74683207416eb2a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/18e660c3b898475da74683207416eb2a 2024-11-25T03:17:02,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0fe55677b719460ba4db407dd57d9b59 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0fe55677b719460ba4db407dd57d9b59 2024-11-25T03:17:02,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/e281a111bb0f4391b795e6e32f6f92fa to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/e281a111bb0f4391b795e6e32f6f92fa 2024-11-25T03:17:02,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/d7ae1e727d7c454283a0ecc0b3c57971 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/d7ae1e727d7c454283a0ecc0b3c57971 2024-11-25T03:17:02,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0ec96aea2f8d4f6a988998616d95853d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0ec96aea2f8d4f6a988998616d95853d 2024-11-25T03:17:02,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/8fdf59517bbe4cdd9ce679cb21da5241 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/8fdf59517bbe4cdd9ce679cb21da5241 2024-11-25T03:17:02,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/9b65d7c8f470427eb65d901ce5a79d1b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/9b65d7c8f470427eb65d901ce5a79d1b 2024-11-25T03:17:02,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cd923c4ef35e4226ae4c6cb5d57061ca to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/cd923c4ef35e4226ae4c6cb5d57061ca 2024-11-25T03:17:02,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/b9d5af3b3b2e4697a1413dfc1581b2c1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/b9d5af3b3b2e4697a1413dfc1581b2c1 2024-11-25T03:17:02,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/4bfabcc15bdf4f0393b4f991aa3e4b85 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/4bfabcc15bdf4f0393b4f991aa3e4b85 2024-11-25T03:17:02,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f98b07c328cd404db2faaf5d14027669 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f98b07c328cd404db2faaf5d14027669 2024-11-25T03:17:02,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a2056110d10f489e90dbb56cb67f8483 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/a2056110d10f489e90dbb56cb67f8483 2024-11-25T03:17:02,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/91fdf62af25b4301a9096619e29cf1fc to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/91fdf62af25b4301a9096619e29cf1fc 2024-11-25T03:17:02,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/96267559dd1d4e5a83a7b722df3e7d33 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/96267559dd1d4e5a83a7b722df3e7d33 2024-11-25T03:17:02,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/462c7ae2b83e4d569e50499e0c60f75a to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/462c7ae2b83e4d569e50499e0c60f75a 2024-11-25T03:17:02,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f5a0340ababe49658881a8b42ee458b9 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/f5a0340ababe49658881a8b42ee458b9 2024-11-25T03:17:02,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0310c7606f0d468a951281cc0a255a37 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/0310c7606f0d468a951281cc0a255a37 2024-11-25T03:17:02,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d151f040a02e4cbfa43e5e5b8ae8052d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/bbc5657ea3334bec8d668cfffb92dd4c, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/575fe36447d14176a8b119306fab35ab, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8877f6afc7504c3fbc523223d5f4e670, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/2404b98c297c4b23a50a58a8c6eb6929, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/55932c66fc6e4a10a142d257dfd903b4, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/02d8a8de5348477b91fb4676b8fb7c6e, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/f0cca49c2ba043119461dbdb8a4385c1, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8882201c01ac44e1ae08ddb4c2eb0dc7, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/73f15b71719f45038a856884ab5ada3d, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/1dcc28e282954cc2a8773dcbc8b478af, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/3a1070939c2c46c192829150ec7cec09, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d5e36729e18d408c9daca60e109c1b9b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/c884ecc264f248bbad0cc1d368de01c0, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/01e529e30e2e4b66a181a9dd3398f613, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/9b93fcfc51db4c98b1c7e63debd43d7b, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/579d495118554ab69cf69df2823331c8, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/fd89a77134bf460d802ee439c916d889, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/a3f52af6f4844ccd8754ab4ce9684f39, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/976ddf7657a448d484df4f10ef20c991, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8108c04e038b400d98d2046c3e55a712] to archive 2024-11-25T03:17:02,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:17:02,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d151f040a02e4cbfa43e5e5b8ae8052d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d151f040a02e4cbfa43e5e5b8ae8052d 2024-11-25T03:17:02,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/bbc5657ea3334bec8d668cfffb92dd4c to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/bbc5657ea3334bec8d668cfffb92dd4c 2024-11-25T03:17:02,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/575fe36447d14176a8b119306fab35ab to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/575fe36447d14176a8b119306fab35ab 2024-11-25T03:17:02,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8877f6afc7504c3fbc523223d5f4e670 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8877f6afc7504c3fbc523223d5f4e670 2024-11-25T03:17:02,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/2404b98c297c4b23a50a58a8c6eb6929 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/2404b98c297c4b23a50a58a8c6eb6929 2024-11-25T03:17:02,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/55932c66fc6e4a10a142d257dfd903b4 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/55932c66fc6e4a10a142d257dfd903b4 2024-11-25T03:17:02,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/02d8a8de5348477b91fb4676b8fb7c6e to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/02d8a8de5348477b91fb4676b8fb7c6e 2024-11-25T03:17:02,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/f0cca49c2ba043119461dbdb8a4385c1 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/f0cca49c2ba043119461dbdb8a4385c1 2024-11-25T03:17:02,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8882201c01ac44e1ae08ddb4c2eb0dc7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8882201c01ac44e1ae08ddb4c2eb0dc7 2024-11-25T03:17:02,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/73f15b71719f45038a856884ab5ada3d to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/73f15b71719f45038a856884ab5ada3d 2024-11-25T03:17:02,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/1dcc28e282954cc2a8773dcbc8b478af to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/1dcc28e282954cc2a8773dcbc8b478af 2024-11-25T03:17:02,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/3a1070939c2c46c192829150ec7cec09 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/3a1070939c2c46c192829150ec7cec09 2024-11-25T03:17:02,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d5e36729e18d408c9daca60e109c1b9b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/d5e36729e18d408c9daca60e109c1b9b 2024-11-25T03:17:02,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/c884ecc264f248bbad0cc1d368de01c0 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/c884ecc264f248bbad0cc1d368de01c0 2024-11-25T03:17:02,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/01e529e30e2e4b66a181a9dd3398f613 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/01e529e30e2e4b66a181a9dd3398f613 2024-11-25T03:17:02,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/9b93fcfc51db4c98b1c7e63debd43d7b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/9b93fcfc51db4c98b1c7e63debd43d7b 2024-11-25T03:17:02,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/579d495118554ab69cf69df2823331c8 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/579d495118554ab69cf69df2823331c8 2024-11-25T03:17:02,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/fd89a77134bf460d802ee439c916d889 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/fd89a77134bf460d802ee439c916d889 2024-11-25T03:17:02,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/a3f52af6f4844ccd8754ab4ce9684f39 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/a3f52af6f4844ccd8754ab4ce9684f39 2024-11-25T03:17:02,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/976ddf7657a448d484df4f10ef20c991 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/976ddf7657a448d484df4f10ef20c991 2024-11-25T03:17:02,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8108c04e038b400d98d2046c3e55a712 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/8108c04e038b400d98d2046c3e55a712 2024-11-25T03:17:02,728 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/recovered.edits/309.seqid, newMaxSeqId=309, maxSeqId=4 2024-11-25T03:17:02,729 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. 2024-11-25T03:17:02,729 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 08a856257124e7cc799bf8e479f21703: Waiting for close lock at 1732504621022Running coprocessor pre-close hooks at 1732504621022Disabling compacts and flushes for region at 1732504621022Disabling writes for close at 1732504621022Obtaining lock to block concurrent updates at 1732504621022Preparing flush snapshotting stores in 08a856257124e7cc799bf8e479f21703 at 1732504621022Finished memstore snapshotting TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732504621022Flushing stores of TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703. at 1732504621022Flushing 08a856257124e7cc799bf8e479f21703/A: creating writer at 1732504621023 (+1 ms)Flushing mob file 08a856257124e7cc799bf8e479f21703/A: appending metadata at 1732504621027 (+4 ms)Flushing mob file 08a856257124e7cc799bf8e479f21703/A: closing flushed file at 1732504621027Flushing 08a856257124e7cc799bf8e479f21703/A: appending metadata at 1732504621434 (+407 ms)Flushing 08a856257124e7cc799bf8e479f21703/A: closing flushed file at 1732504621434Flushing 08a856257124e7cc799bf8e479f21703/B: creating writer at 1732504621845 (+411 ms)Flushing 08a856257124e7cc799bf8e479f21703/B: appending metadata at 1732504621847 (+2 ms)Flushing 08a856257124e7cc799bf8e479f21703/B: closing flushed file at 1732504621847Flushing 08a856257124e7cc799bf8e479f21703/C: creating writer at 1732504622253 (+406 ms)Flushing 08a856257124e7cc799bf8e479f21703/C: appending metadata at 1732504622255 (+2 ms)Flushing 08a856257124e7cc799bf8e479f21703/C: closing flushed file at 1732504622255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe48721: reopening flushed file at 1732504622662 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@320ff4f2: reopening flushed file at 1732504622665 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ddbf68b: reopening flushed file at 1732504622668 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 08a856257124e7cc799bf8e479f21703 in 1649ms, sequenceid=306, compaction requested=false at 1732504622671 (+3 ms)Writing region close event to WAL at 1732504622725 (+54 ms)Running coprocessor post-close hooks at 1732504622728 (+3 ms)Closed at 1732504622729 (+1 ms) 2024-11-25T03:17:02,730 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:02,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=08a856257124e7cc799bf8e479f21703, regionState=CLOSED 2024-11-25T03:17:02,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 because future has completed 2024-11-25T03:17:02,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-25T03:17:02,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 08a856257124e7cc799bf8e479f21703, server=38a5d2077268,43357,1732504425108 in 1.8630 sec 2024-11-25T03:17:02,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-25T03:17:02,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=08a856257124e7cc799bf8e479f21703, UNASSIGN in 1.8670 sec 2024-11-25T03:17:02,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-25T03:17:02,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8690 sec 2024-11-25T03:17:02,738 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732504622737"}]},"ts":"1732504622737"} 2024-11-25T03:17:02,739 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:17:02,739 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:17:02,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-25T03:17:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-25T03:17:03,010 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:17:03,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:17:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:17:03,012 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:17:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-25T03:17:03,013 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:17:03,015 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,017 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C, FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/recovered.edits] 2024-11-25T03:17:03,019 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8bd260fc1ce84dfb8b671be07f748fda to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/8bd260fc1ce84dfb8b671be07f748fda 2024-11-25T03:17:03,020 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/901d18d29633482b85dadd7b9d616277 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/A/901d18d29633482b85dadd7b9d616277 2024-11-25T03:17:03,021 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/6d8d9dcd722b44e5beaf53c6d0cfd53b to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/6d8d9dcd722b44e5beaf53c6d0cfd53b 2024-11-25T03:17:03,022 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/7e70a4d73680448dbec91904deee9b4f to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/B/7e70a4d73680448dbec91904deee9b4f 2024-11-25T03:17:03,024 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/52ed38e0f25d4d7ebddca70afc04ecb7 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/52ed38e0f25d4d7ebddca70afc04ecb7 2024-11-25T03:17:03,025 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/900adac488e3464bbeee54d7024a52fd to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/C/900adac488e3464bbeee54d7024a52fd 2024-11-25T03:17:03,027 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/recovered.edits/309.seqid to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703/recovered.edits/309.seqid 2024-11-25T03:17:03,027 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/default/TestAcidGuarantees/08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,027 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:17:03,027 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:17:03,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T03:17:03,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250126a6623e7342aab00c76c235da9caf_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250126a6623e7342aab00c76c235da9caf_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,031 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112501924ece69e64042a2581d30d6284445_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112501924ece69e64042a2581d30d6284445_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,032 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112505083e88d04d4711b38e75bdecfc778c_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112505083e88d04d4711b38e75bdecfc778c_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253145b60296f247dd9d5da0afeef20fea_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253145b60296f247dd9d5da0afeef20fea_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125522fd138bfc34951a3c17d70d38460eb_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125522fd138bfc34951a3c17d70d38460eb_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112561d6ce901103492b85cb3363f043fed4_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112561d6ce901103492b85cb3363f043fed4_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ef2ccf5e02143d59c973de67dea1e74_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ef2ccf5e02143d59c973de67dea1e74_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112580c35cffad9041268d0cd3e7029a7089_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112580c35cffad9041268d0cd3e7029a7089_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125912e0c2dc1a648e6a38d38dd92638de5_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125912e0c2dc1a648e6a38d38dd92638de5_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a9d603f10274497d9ea209f5230bbcfa_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a9d603f10274497d9ea209f5230bbcfa_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aa590b1c84224a1f9cc85d5f9154a233_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aa590b1c84224a1f9cc85d5f9154a233_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,040 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aaf9832233bb4aacb2dbb8d971bbd335_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aaf9832233bb4aacb2dbb8d971bbd335_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b2a45c27c5bd41bd802d5da105d124ef_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b2a45c27c5bd41bd802d5da105d124ef_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c9f54505417a4b388665351781ab2663_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c9f54505417a4b388665351781ab2663_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,043 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ca0e7c81d4014f7c90a8b73cddf19060_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ca0e7c81d4014f7c90a8b73cddf19060_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,043 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d1a73ed115ac4eceb89c06e5a3ab27b9_08a856257124e7cc799bf8e479f21703 to hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d1a73ed115ac4eceb89c06e5a3ab27b9_08a856257124e7cc799bf8e479f21703 2024-11-25T03:17:03,044 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:17:03,045 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:17:03,048 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:17:03,049 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:17:03,050 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:17:03,050 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:17:03,051 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732504623050"}]},"ts":"9223372036854775807"} 2024-11-25T03:17:03,052 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:17:03,052 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 08a856257124e7cc799bf8e479f21703, NAME => 'TestAcidGuarantees,,1732504586619.08a856257124e7cc799bf8e479f21703.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:17:03,052 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:17:03,052 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732504623052"}]},"ts":"9223372036854775807"} 2024-11-25T03:17:03,054 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:17:03,054 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:17:03,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-25T03:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-25T03:17:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:17:03,120 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:17:03,129 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 250), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=302 (was 382), ProcessCount=11 (was 11), AvailableMemoryMB=6160 (was 6273) 2024-11-25T03:17:03,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:17:03,129 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:17:03,129 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:17:03,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:03,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:03,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:17:03,130 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:17:03,130 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-25T03:17:03,130 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=38a5d2077268,44623,1732504424313 2024-11-25T03:17:03,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:17:03,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:17:03,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:17:03,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:17:03,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:17:03,132 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:17:03,132 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:17:03,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:03,133 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '38a5d2077268,43357,1732504425108' ***** 2024-11-25T03:17:03,133 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:17:03,133 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:17:03,133 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:17:03,133 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:17:03,134 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(959): stopping server 38a5d2077268,43357,1732504425108 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;38a5d2077268:43357. 2024-11-25T03:17:03,134 DEBUG [RS:0;38a5d2077268:43357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:17:03,134 DEBUG [RS:0;38a5d2077268:43357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:17:03,134 INFO [RS:0;38a5d2077268:43357 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:17:03,135 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:17:03,135 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T03:17:03,135 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T03:17:03,135 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:03,135 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:17:03,135 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:17:03,135 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:17:03,136 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:17:03,136 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:17:03,136 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-25T03:17:03,157 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/info/6cf40652878a404580b5594361b607e8 is 95, key is TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f./info:/1732504460640/DeleteFamily/seqid=0 2024-11-25T03:17:03,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742428_1604 (size=6514) 2024-11-25T03:17:03,173 INFO [regionserver/38a5d2077268:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:17:03,257 INFO [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T03:17:03,257 INFO [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T03:17:03,336 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:03,536 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:03,561 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/info/6cf40652878a404580b5594361b607e8 2024-11-25T03:17:03,578 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/ns/1f0641ee2cac4b149bbdecc6445b6ef3 is 93, key is TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f./ns:/1732504460588/DeleteFamily/seqid=0 2024-11-25T03:17:03,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742429_1605 (size=5906) 2024-11-25T03:17:03,736 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:03,936 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:03,982 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/ns/1f0641ee2cac4b149bbdecc6445b6ef3 2024-11-25T03:17:03,999 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/rep_barrier/91cfd3fb224c45349ab0ba9f60e24ea3 is 102, key is TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f./rep_barrier:/1732504460588/DeleteFamily/seqid=0 2024-11-25T03:17:04,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742430_1606 (size=6025) 2024-11-25T03:17:04,136 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:04,337 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:04,403 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/rep_barrier/91cfd3fb224c45349ab0ba9f60e24ea3 2024-11-25T03:17:04,420 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/table/cf4914c83b234610abb576f878e74074 is 96, key is TestAcidGuarantees,,1732504428483.159c9b03400b7fa4e53f5245dc0f2c9f./table:/1732504460588/DeleteFamily/seqid=0 2024-11-25T03:17:04,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742431_1607 (size=5936) 2024-11-25T03:17:04,537 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:04,737 DEBUG [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:17:04,756 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T03:17:04,823 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/table/cf4914c83b234610abb576f878e74074 2024-11-25T03:17:04,826 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/info/6cf40652878a404580b5594361b607e8 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/info/6cf40652878a404580b5594361b607e8 2024-11-25T03:17:04,829 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/info/6cf40652878a404580b5594361b607e8, entries=12, sequenceid=90, filesize=6.4 K 2024-11-25T03:17:04,830 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/ns/1f0641ee2cac4b149bbdecc6445b6ef3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/ns/1f0641ee2cac4b149bbdecc6445b6ef3 2024-11-25T03:17:04,832 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/ns/1f0641ee2cac4b149bbdecc6445b6ef3, entries=8, sequenceid=90, filesize=5.8 K 2024-11-25T03:17:04,833 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/rep_barrier/91cfd3fb224c45349ab0ba9f60e24ea3 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/rep_barrier/91cfd3fb224c45349ab0ba9f60e24ea3 2024-11-25T03:17:04,835 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/rep_barrier/91cfd3fb224c45349ab0ba9f60e24ea3, entries=6, sequenceid=90, filesize=5.9 K 2024-11-25T03:17:04,836 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/.tmp/table/cf4914c83b234610abb576f878e74074 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/table/cf4914c83b234610abb576f878e74074 2024-11-25T03:17:04,838 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/table/cf4914c83b234610abb576f878e74074, entries=7, sequenceid=90, filesize=5.8 K 2024-11-25T03:17:04,839 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false 2024-11-25T03:17:04,842 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-25T03:17:04,843 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:17:04,843 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:17:04,843 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732504623135Running coprocessor pre-close hooks at 1732504623135Disabling compacts and flushes for region at 1732504623135Disabling writes for close at 1732504623136 (+1 ms)Obtaining lock to block concurrent updates at 1732504623136Preparing flush snapshotting stores in 1588230740 at 1732504623136Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732504623136Flushing stores of hbase:meta,,1.1588230740 at 1732504623137 (+1 ms)Flushing 1588230740/info: creating writer at 1732504623137Flushing 1588230740/info: appending metadata at 1732504623157 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732504623157Flushing 1588230740/ns: creating writer at 1732504623564 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732504623578 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732504623578Flushing 1588230740/rep_barrier: creating writer at 1732504623985 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732504623999 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732504623999Flushing 1588230740/table: creating writer at 1732504624406 (+407 ms)Flushing 1588230740/table: appending metadata at 1732504624420 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732504624420Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c113d82: reopening flushed file at 1732504624826 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47b6e156: reopening flushed file at 1732504624829 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d1e6242: reopening flushed file at 1732504624833 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2af583ec: reopening flushed file at 1732504624836 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false at 1732504624839 (+3 ms)Writing region close event to WAL at 1732504624840 (+1 ms)Running coprocessor post-close hooks at 1732504624843 (+3 ms)Closed at 1732504624843 2024-11-25T03:17:04,843 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:17:04,937 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(976): stopping server 38a5d2077268,43357,1732504425108; all regions closed. 2024-11-25T03:17:04,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741834_1010 (size=25344) 2024-11-25T03:17:04,944 DEBUG [RS:0;38a5d2077268:43357 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/oldWALs 2024-11-25T03:17:04,944 INFO [RS:0;38a5d2077268:43357 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 38a5d2077268%2C43357%2C1732504425108.meta:.meta(num 1732504427943) 2024-11-25T03:17:04,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741833_1009 (size=16012168) 2024-11-25T03:17:04,947 DEBUG [RS:0;38a5d2077268:43357 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/oldWALs 2024-11-25T03:17:04,947 INFO [RS:0;38a5d2077268:43357 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 38a5d2077268%2C43357%2C1732504425108:(num 1732504427418) 2024-11-25T03:17:04,947 DEBUG [RS:0;38a5d2077268:43357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:17:04,947 INFO [RS:0;38a5d2077268:43357 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:17:04,947 INFO [RS:0;38a5d2077268:43357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:17:04,947 INFO [RS:0;38a5d2077268:43357 {}] hbase.ChoreService(370): Chore service for: regionserver/38a5d2077268:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T03:17:04,947 INFO [RS:0;38a5d2077268:43357 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:17:04,948 INFO [regionserver/38a5d2077268:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:17:04,948 INFO [RS:0;38a5d2077268:43357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43357 2024-11-25T03:17:04,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:17:04,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/38a5d2077268,43357,1732504425108 2024-11-25T03:17:04,958 INFO [RS:0;38a5d2077268:43357 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:17:04,967 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [38a5d2077268,43357,1732504425108] 2024-11-25T03:17:04,971 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/38a5d2077268,43357,1732504425108 already deleted, retry=false 2024-11-25T03:17:04,971 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 38a5d2077268,43357,1732504425108 expired; onlineServers=0 2024-11-25T03:17:04,971 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '38a5d2077268,44623,1732504424313' ***** 2024-11-25T03:17:04,971 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:17:04,971 INFO [M:0;38a5d2077268:44623 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:17:04,972 INFO [M:0;38a5d2077268:44623 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:17:04,972 DEBUG [M:0;38a5d2077268:44623 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:17:04,972 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:17:04,972 DEBUG [M:0;38a5d2077268:44623 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:17:04,972 DEBUG [master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732504427075 {}] cleaner.HFileCleaner(306): Exit Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732504427075,5,FailOnTimeoutGroup] 2024-11-25T03:17:04,972 DEBUG [master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732504427076 {}] cleaner.HFileCleaner(306): Exit Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732504427076,5,FailOnTimeoutGroup] 2024-11-25T03:17:04,972 INFO [M:0;38a5d2077268:44623 {}] hbase.ChoreService(370): Chore service for: master/38a5d2077268:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:17:04,972 INFO [M:0;38a5d2077268:44623 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:17:04,972 DEBUG [M:0;38a5d2077268:44623 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:17:04,972 INFO [M:0;38a5d2077268:44623 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:17:04,972 INFO [M:0;38a5d2077268:44623 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:17:04,973 ERROR [M:0;38a5d2077268:44623 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:32883 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:32883,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-25T03:17:04,973 INFO [M:0;38a5d2077268:44623 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:17:04,973 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:17:04,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:17:04,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:17:04,974 DEBUG [M:0;38a5d2077268:44623 {}] zookeeper.ZKUtil(347): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:17:04,974 WARN [M:0;38a5d2077268:44623 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:17:04,974 INFO [M:0;38a5d2077268:44623 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/.lastflushedseqids 2024-11-25T03:17:04,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742432_1608 (size=72) 2024-11-25T03:17:05,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:17:05,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43357-0x1014d1d75e90001, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:17:05,068 INFO [RS:0;38a5d2077268:43357 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:17:05,068 INFO [RS:0;38a5d2077268:43357 {}] regionserver.HRegionServer(1031): Exiting; stopping=38a5d2077268,43357,1732504425108; zookeeper connection closed. 2024-11-25T03:17:05,068 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4bf22f91 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4bf22f91 2024-11-25T03:17:05,069 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T03:17:05,384 INFO [M:0;38a5d2077268:44623 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:17:05,384 INFO [M:0;38a5d2077268:44623 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:17:05,384 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:17:05,385 INFO [M:0;38a5d2077268:44623 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:17:05,385 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:17:05,385 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:17:05,385 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:17:05,385 INFO [M:0;38a5d2077268:44623 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=791.43 KB heapSize=970.74 KB 2024-11-25T03:17:05,400 DEBUG [M:0;38a5d2077268:44623 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f46b5dd389c0499c8822964b386156db is 82, key is hbase:meta,,1/info:regioninfo/1732504428053/Put/seqid=0 2024-11-25T03:17:05,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742433_1609 (size=5672) 2024-11-25T03:17:05,804 INFO [M:0;38a5d2077268:44623 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2218 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f46b5dd389c0499c8822964b386156db 2024-11-25T03:17:05,823 DEBUG [M:0;38a5d2077268:44623 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a027f9fbbc6248d8ace806138ae5fae1 is 2479, key is \x00\x00\x00\x00\x00\x00\x00'/proc:d/1732504463995/Put/seqid=0 2024-11-25T03:17:05,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742434_1610 (size=46043) 2024-11-25T03:17:06,227 INFO [M:0;38a5d2077268:44623 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=790.88 KB at sequenceid=2218 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a027f9fbbc6248d8ace806138ae5fae1 2024-11-25T03:17:06,230 INFO [M:0;38a5d2077268:44623 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a027f9fbbc6248d8ace806138ae5fae1 2024-11-25T03:17:06,245 DEBUG [M:0;38a5d2077268:44623 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8505f6e6f52a4e428f653ca4e464ec80 is 69, key is 38a5d2077268,43357,1732504425108/rs:state/1732504427183/Put/seqid=0 2024-11-25T03:17:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073742435_1611 (size=5156) 2024-11-25T03:17:06,648 INFO [M:0;38a5d2077268:44623 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2218 (bloomFilter=true), to=hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8505f6e6f52a4e428f653ca4e464ec80 2024-11-25T03:17:06,652 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f46b5dd389c0499c8822964b386156db as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f46b5dd389c0499c8822964b386156db 2024-11-25T03:17:06,654 INFO [M:0;38a5d2077268:44623 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f46b5dd389c0499c8822964b386156db, entries=8, sequenceid=2218, filesize=5.5 K 2024-11-25T03:17:06,655 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a027f9fbbc6248d8ace806138ae5fae1 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a027f9fbbc6248d8ace806138ae5fae1 2024-11-25T03:17:06,657 INFO [M:0;38a5d2077268:44623 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a027f9fbbc6248d8ace806138ae5fae1 2024-11-25T03:17:06,657 INFO [M:0;38a5d2077268:44623 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a027f9fbbc6248d8ace806138ae5fae1, entries=174, sequenceid=2218, filesize=45.0 K 2024-11-25T03:17:06,658 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8505f6e6f52a4e428f653ca4e464ec80 as hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8505f6e6f52a4e428f653ca4e464ec80 2024-11-25T03:17:06,660 INFO [M:0;38a5d2077268:44623 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32883/user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8505f6e6f52a4e428f653ca4e464ec80, entries=1, sequenceid=2218, filesize=5.0 K 2024-11-25T03:17:06,661 INFO [M:0;38a5d2077268:44623 {}] regionserver.HRegion(3140): Finished flush of dataSize ~791.43 KB/810427, heapSize ~970.45 KB/993736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2218, compaction requested=false 2024-11-25T03:17:06,662 INFO [M:0;38a5d2077268:44623 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:17:06,662 DEBUG [M:0;38a5d2077268:44623 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732504625384Disabling compacts and flushes for region at 1732504625384Disabling writes for close at 1732504625385 (+1 ms)Obtaining lock to block concurrent updates at 1732504625385Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732504625385Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=810427, getHeapSize=993976, getOffHeapSize=0, getCellsCount=2417 at 1732504625385Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732504625386 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732504625386Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732504625400 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732504625400Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732504625807 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732504625823 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732504625823Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732504626230 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732504626244 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732504626244Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ea01fd: reopening flushed file at 1732504626651 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@508eb45c: reopening flushed file at 1732504626654 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43eb13e: reopening flushed file at 1732504626657 (+3 ms)Finished flush of dataSize ~791.43 KB/810427, heapSize ~970.45 KB/993736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2218, compaction requested=false at 1732504626661 (+4 ms)Writing region close event to WAL at 1732504626662 (+1 ms)Closed at 1732504626662 2024-11-25T03:17:06,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42607 is added to blk_1073741830_1006 (size=955812) 2024-11-25T03:17:06,664 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/1103d505-03d5-70cc-1a21-d54fd6f3b0bf/MasterData/WALs/38a5d2077268,44623,1732504424313/38a5d2077268%2C44623%2C1732504424313.1732504426435 not finished, retry = 0 2024-11-25T03:17:06,765 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:17:06,765 INFO [M:0;38a5d2077268:44623 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T03:17:06,766 INFO [M:0;38a5d2077268:44623 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44623 2024-11-25T03:17:06,766 INFO [M:0;38a5d2077268:44623 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:17:06,868 INFO [M:0;38a5d2077268:44623 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:17:06,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:17:06,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44623-0x1014d1d75e90000, quorum=127.0.0.1:58288, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:17:06,873 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:17:06,876 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:17:06,876 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:17:06,876 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:17:06,876 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/hadoop.log.dir/,STOPPED} 2024-11-25T03:17:06,879 WARN [BP-37819047-172.17.0.2-1732504421123 heartbeating to localhost/127.0.0.1:32883 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T03:17:06,879 WARN [BP-37819047-172.17.0.2-1732504421123 heartbeating to localhost/127.0.0.1:32883 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-37819047-172.17.0.2-1732504421123 (Datanode Uuid 8fb8ee07-1675-42fa-a32b-ac65b0099908) service to localhost/127.0.0.1:32883 2024-11-25T03:17:06,879 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T03:17:06,879 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T03:17:06,882 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/data/data1/current/BP-37819047-172.17.0.2-1732504421123 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:17:06,882 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/cluster_ce8b57d6-599a-49d7-cd12-fd61b014a41a/data/data2/current/BP-37819047-172.17.0.2-1732504421123 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:17:06,882 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T03:17:06,890 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:17:06,891 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:17:06,891 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:17:06,891 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:17:06,891 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d439e825-62ac-17c0-1cfb-865fec255fcd/hadoop.log.dir/,STOPPED} 2024-11-25T03:17:06,906 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T03:17:07,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down